JobList.php 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464
  1. <?php
  2. /**
  3. * @copyright Copyright (c) 2016, ownCloud, Inc.
  4. *
  5. * @author Christoph Wurst <christoph@winzerhof-wurst.at>
  6. * @author Georg Ehrke <oc.list@georgehrke.com>
  7. * @author Joas Schilling <coding@schilljs.com>
  8. * @author Jörn Friedrich Dreyer <jfd@butonic.de>
  9. * @author Morris Jobke <hey@morrisjobke.de>
  10. * @author Noveen Sachdeva <noveen.sachdeva@research.iiit.ac.in>
  11. * @author Robin Appelman <robin@icewind.nl>
  12. * @author Robin McCorkell <robin@mccorkell.me.uk>
  13. * @author Roeland Jago Douma <roeland@famdouma.nl>
  14. *
  15. * @license AGPL-3.0
  16. *
  17. * This code is free software: you can redistribute it and/or modify
  18. * it under the terms of the GNU Affero General Public License, version 3,
  19. * as published by the Free Software Foundation.
  20. *
  21. * This program is distributed in the hope that it will be useful,
  22. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  23. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  24. * GNU Affero General Public License for more details.
  25. *
  26. * You should have received a copy of the GNU Affero General Public License, version 3,
  27. * along with this program. If not, see <http://www.gnu.org/licenses/>
  28. *
  29. */
  30. namespace OC\BackgroundJob;
  31. use Doctrine\DBAL\Platforms\MySQLPlatform;
  32. use OCP\AppFramework\QueryException;
  33. use OCP\AppFramework\Utility\ITimeFactory;
  34. use OCP\AutoloadNotAllowedException;
  35. use OCP\BackgroundJob\IJob;
  36. use OCP\BackgroundJob\IJobList;
  37. use OCP\BackgroundJob\IParallelAwareJob;
  38. use OCP\DB\Exception;
  39. use OCP\DB\QueryBuilder\IQueryBuilder;
  40. use OCP\IConfig;
  41. use OCP\IDBConnection;
  42. use Psr\Log\LoggerInterface;
  43. use function get_class;
  44. use function json_encode;
  45. use function md5;
  46. use function strlen;
  47. class JobList implements IJobList {
  48. protected IDBConnection $connection;
  49. protected IConfig $config;
  50. protected ITimeFactory $timeFactory;
  51. protected LoggerInterface $logger;
  52. public function __construct(IDBConnection $connection, IConfig $config, ITimeFactory $timeFactory, LoggerInterface $logger) {
  53. $this->connection = $connection;
  54. $this->config = $config;
  55. $this->timeFactory = $timeFactory;
  56. $this->logger = $logger;
  57. }
  58. public function add($job, $argument = null, ?int $firstCheck = null): void {
  59. if ($firstCheck === null) {
  60. $firstCheck = $this->timeFactory->getTime();
  61. }
  62. if ($job instanceof IJob) {
  63. $class = get_class($job);
  64. } else {
  65. $class = $job;
  66. }
  67. $argumentJson = json_encode($argument);
  68. if (strlen($argumentJson) > 4000) {
  69. throw new \InvalidArgumentException('Background job arguments can\'t exceed 4000 characters (json encoded)');
  70. }
  71. $query = $this->connection->getQueryBuilder();
  72. if (!$this->has($job, $argument)) {
  73. $query->insert('jobs')
  74. ->values([
  75. 'class' => $query->createNamedParameter($class),
  76. 'argument' => $query->createNamedParameter($argumentJson),
  77. 'argument_hash' => $query->createNamedParameter(md5($argumentJson)),
  78. 'last_run' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
  79. 'last_checked' => $query->createNamedParameter($firstCheck, IQueryBuilder::PARAM_INT),
  80. ]);
  81. } else {
  82. $query->update('jobs')
  83. ->set('reserved_at', $query->expr()->literal(0, IQueryBuilder::PARAM_INT))
  84. ->set('last_checked', $query->createNamedParameter($firstCheck, IQueryBuilder::PARAM_INT))
  85. ->set('last_run', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))
  86. ->where($query->expr()->eq('class', $query->createNamedParameter($class)))
  87. ->andWhere($query->expr()->eq('argument_hash', $query->createNamedParameter(md5($argumentJson))));
  88. }
  89. $query->executeStatement();
  90. }
  91. public function scheduleAfter(string $job, int $runAfter, $argument = null): void {
  92. $this->add($job, $argument, $runAfter);
  93. }
  94. /**
  95. * @param IJob|string $job
  96. * @param mixed $argument
  97. */
  98. public function remove($job, $argument = null): void {
  99. if ($job instanceof IJob) {
  100. $class = get_class($job);
  101. } else {
  102. $class = $job;
  103. }
  104. $query = $this->connection->getQueryBuilder();
  105. $query->delete('jobs')
  106. ->where($query->expr()->eq('class', $query->createNamedParameter($class)));
  107. if (!is_null($argument)) {
  108. $argumentJson = json_encode($argument);
  109. $query->andWhere($query->expr()->eq('argument_hash', $query->createNamedParameter(md5($argumentJson))));
  110. }
  111. // Add galera safe delete chunking if using mysql
  112. // Stops us hitting wsrep_max_ws_rows when large row counts are deleted
  113. if ($this->connection->getDatabasePlatform() instanceof MySQLPlatform) {
  114. // Then use chunked delete
  115. $max = IQueryBuilder::MAX_ROW_DELETION;
  116. $query->setMaxResults($max);
  117. do {
  118. $deleted = $query->execute();
  119. } while ($deleted === $max);
  120. } else {
  121. // Dont use chunked delete - let the DB handle the large row count natively
  122. $query->execute();
  123. }
  124. }
  125. protected function removeById(int $id): void {
  126. $query = $this->connection->getQueryBuilder();
  127. $query->delete('jobs')
  128. ->where($query->expr()->eq('id', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
  129. $query->executeStatement();
  130. }
  131. /**
  132. * check if a job is in the list
  133. *
  134. * @param IJob|class-string<IJob> $job
  135. * @param mixed $argument
  136. */
  137. public function has($job, $argument): bool {
  138. if ($job instanceof IJob) {
  139. $class = get_class($job);
  140. } else {
  141. $class = $job;
  142. }
  143. $argument = json_encode($argument);
  144. $query = $this->connection->getQueryBuilder();
  145. $query->select('id')
  146. ->from('jobs')
  147. ->where($query->expr()->eq('class', $query->createNamedParameter($class)))
  148. ->andWhere($query->expr()->eq('argument_hash', $query->createNamedParameter(md5($argument))))
  149. ->setMaxResults(1);
  150. $result = $query->executeQuery();
  151. $row = $result->fetch();
  152. $result->closeCursor();
  153. return (bool) $row;
  154. }
  155. public function getJobs($job, ?int $limit, int $offset): array {
  156. $iterable = $this->getJobsIterator($job, $limit, $offset);
  157. if (is_array($iterable)) {
  158. return $iterable;
  159. } else {
  160. return iterator_to_array($iterable);
  161. }
  162. }
  163. /**
  164. * @param IJob|class-string<IJob>|null $job
  165. * @return iterable<IJob> Avoid to store these objects as they may share a Singleton instance. You should instead use these IJobs instances while looping on the iterable.
  166. */
  167. public function getJobsIterator($job, ?int $limit, int $offset): iterable {
  168. $query = $this->connection->getQueryBuilder();
  169. $query->select('*')
  170. ->from('jobs')
  171. ->setMaxResults($limit)
  172. ->setFirstResult($offset);
  173. if ($job !== null) {
  174. if ($job instanceof IJob) {
  175. $class = get_class($job);
  176. } else {
  177. $class = $job;
  178. }
  179. $query->where($query->expr()->eq('class', $query->createNamedParameter($class)));
  180. }
  181. $result = $query->executeQuery();
  182. while ($row = $result->fetch()) {
  183. $job = $this->buildJob($row);
  184. if ($job) {
  185. yield $job;
  186. }
  187. }
  188. $result->closeCursor();
  189. }
  190. /**
  191. * @inheritDoc
  192. */
  193. public function getNext(bool $onlyTimeSensitive = false, ?array $jobClasses = null): ?IJob {
  194. $query = $this->connection->getQueryBuilder();
  195. $query->select('*')
  196. ->from('jobs')
  197. ->where($query->expr()->lte('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - 12 * 3600, IQueryBuilder::PARAM_INT)))
  198. ->andWhere($query->expr()->lte('last_checked', $query->createNamedParameter($this->timeFactory->getTime(), IQueryBuilder::PARAM_INT)))
  199. ->orderBy('last_checked', 'ASC')
  200. ->setMaxResults(1);
  201. if ($onlyTimeSensitive) {
  202. $query->andWhere($query->expr()->eq('time_sensitive', $query->createNamedParameter(IJob::TIME_SENSITIVE, IQueryBuilder::PARAM_INT)));
  203. }
  204. if ($jobClasses !== null && count($jobClasses) > 0) {
  205. $orClasses = $query->expr()->orx();
  206. foreach ($jobClasses as $jobClass) {
  207. $orClasses->add($query->expr()->eq('class', $query->createNamedParameter($jobClass, IQueryBuilder::PARAM_STR)));
  208. }
  209. $query->andWhere($orClasses);
  210. }
  211. $result = $query->executeQuery();
  212. $row = $result->fetch();
  213. $result->closeCursor();
  214. if ($row) {
  215. $job = $this->buildJob($row);
  216. if ($job instanceof IParallelAwareJob && !$job->getAllowParallelRuns() && $this->hasReservedJob(get_class($job))) {
  217. $this->logger->debug('Skipping ' . get_class($job) . ' job with ID ' . $job->getId() . ' because another job with the same class is already running', ['app' => 'cron']);
  218. $update = $this->connection->getQueryBuilder();
  219. $update->update('jobs')
  220. ->set('last_checked', $update->createNamedParameter($this->timeFactory->getTime() + 1))
  221. ->where($update->expr()->eq('id', $update->createParameter('jobid')));
  222. $update->setParameter('jobid', $row['id']);
  223. $update->executeStatement();
  224. return $this->getNext($onlyTimeSensitive);
  225. }
  226. $update = $this->connection->getQueryBuilder();
  227. $update->update('jobs')
  228. ->set('reserved_at', $update->createNamedParameter($this->timeFactory->getTime()))
  229. ->set('last_checked', $update->createNamedParameter($this->timeFactory->getTime()))
  230. ->where($update->expr()->eq('id', $update->createParameter('jobid')))
  231. ->andWhere($update->expr()->eq('reserved_at', $update->createParameter('reserved_at')))
  232. ->andWhere($update->expr()->eq('last_checked', $update->createParameter('last_checked')));
  233. $update->setParameter('jobid', $row['id']);
  234. $update->setParameter('reserved_at', $row['reserved_at']);
  235. $update->setParameter('last_checked', $row['last_checked']);
  236. $count = $update->executeStatement();
  237. if ($count === 0) {
  238. // Background job already executed elsewhere, try again.
  239. return $this->getNext($onlyTimeSensitive, $jobClasses);
  240. }
  241. if ($job === null) {
  242. // set the last_checked to 12h in the future to not check failing jobs all over again
  243. $reset = $this->connection->getQueryBuilder();
  244. $reset->update('jobs')
  245. ->set('reserved_at', $reset->expr()->literal(0, IQueryBuilder::PARAM_INT))
  246. ->set('last_checked', $reset->createNamedParameter($this->timeFactory->getTime() + 12 * 3600, IQueryBuilder::PARAM_INT))
  247. ->where($reset->expr()->eq('id', $reset->createNamedParameter($row['id'], IQueryBuilder::PARAM_INT)));
  248. $reset->executeStatement();
  249. // Background job from disabled app, try again.
  250. return $this->getNext($onlyTimeSensitive, $jobClasses);
  251. }
  252. return $job;
  253. } else {
  254. return null;
  255. }
  256. }
  257. /**
  258. * @return ?IJob The job matching the id. Beware that this object may be a singleton and may be modified by the next call to buildJob.
  259. */
  260. public function getById(int $id): ?IJob {
  261. $row = $this->getDetailsById($id);
  262. if ($row) {
  263. return $this->buildJob($row);
  264. }
  265. return null;
  266. }
  267. public function getDetailsById(int $id): ?array {
  268. $query = $this->connection->getQueryBuilder();
  269. $query->select('*')
  270. ->from('jobs')
  271. ->where($query->expr()->eq('id', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
  272. $result = $query->executeQuery();
  273. $row = $result->fetch();
  274. $result->closeCursor();
  275. if ($row) {
  276. return $row;
  277. }
  278. return null;
  279. }
  280. /**
  281. * get the job object from a row in the db
  282. *
  283. * @param array{class:class-string<IJob>, id:mixed, last_run:mixed, argument:string} $row
  284. * @return ?IJob the next job to run. Beware that this object may be a singleton and may be modified by the next call to buildJob.
  285. */
  286. private function buildJob(array $row): ?IJob {
  287. try {
  288. try {
  289. // Try to load the job as a service
  290. /** @var IJob $job */
  291. $job = \OCP\Server::get($row['class']);
  292. } catch (QueryException $e) {
  293. if (class_exists($row['class'])) {
  294. $class = $row['class'];
  295. $job = new $class();
  296. } else {
  297. // Remove job from disabled app or old version of an app
  298. $this->removeById($row['id']);
  299. return null;
  300. }
  301. }
  302. if (!($job instanceof IJob)) {
  303. // This most likely means an invalid job was enqueued. We can ignore it.
  304. return null;
  305. }
  306. $job->setId((int) $row['id']);
  307. $job->setLastRun((int) $row['last_run']);
  308. $job->setArgument(json_decode($row['argument'], true));
  309. return $job;
  310. } catch (AutoloadNotAllowedException $e) {
  311. // job is from a disabled app, ignore
  312. return null;
  313. }
  314. }
  315. /**
  316. * set the job that was last ran
  317. */
  318. public function setLastJob(IJob $job): void {
  319. $this->unlockJob($job);
  320. $this->config->setAppValue('backgroundjob', 'lastjob', (string)$job->getId());
  321. }
  322. /**
  323. * Remove the reservation for a job
  324. */
  325. public function unlockJob(IJob $job): void {
  326. $query = $this->connection->getQueryBuilder();
  327. $query->update('jobs')
  328. ->set('reserved_at', $query->expr()->literal(0, IQueryBuilder::PARAM_INT))
  329. ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId(), IQueryBuilder::PARAM_INT)));
  330. $query->executeStatement();
  331. }
  332. /**
  333. * set the lastRun of $job to now
  334. */
  335. public function setLastRun(IJob $job): void {
  336. $query = $this->connection->getQueryBuilder();
  337. $query->update('jobs')
  338. ->set('last_run', $query->createNamedParameter(time(), IQueryBuilder::PARAM_INT))
  339. ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId(), IQueryBuilder::PARAM_INT)));
  340. if ($job instanceof \OCP\BackgroundJob\TimedJob
  341. && !$job->isTimeSensitive()) {
  342. $query->set('time_sensitive', $query->createNamedParameter(IJob::TIME_INSENSITIVE));
  343. }
  344. $query->executeStatement();
  345. }
  346. /**
  347. * @param int $timeTaken
  348. */
  349. public function setExecutionTime(IJob $job, $timeTaken): void {
  350. $query = $this->connection->getQueryBuilder();
  351. $query->update('jobs')
  352. ->set('execution_duration', $query->createNamedParameter($timeTaken, IQueryBuilder::PARAM_INT))
  353. ->set('reserved_at', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))
  354. ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId(), IQueryBuilder::PARAM_INT)));
  355. $query->executeStatement();
  356. }
  357. /**
  358. * Reset the $job so it executes on the next trigger
  359. *
  360. * @since 23.0.0
  361. */
  362. public function resetBackgroundJob(IJob $job): void {
  363. $query = $this->connection->getQueryBuilder();
  364. $query->update('jobs')
  365. ->set('last_run', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))
  366. ->set('reserved_at', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))
  367. ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId()), IQueryBuilder::PARAM_INT));
  368. $query->executeStatement();
  369. }
  370. public function hasReservedJob(?string $className = null): bool {
  371. $query = $this->connection->getQueryBuilder();
  372. $query->select('*')
  373. ->from('jobs')
  374. ->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - 6 * 3600, IQueryBuilder::PARAM_INT)))
  375. ->setMaxResults(1);
  376. if ($className !== null) {
  377. $query->andWhere($query->expr()->eq('class', $query->createNamedParameter($className)));
  378. }
  379. try {
  380. $result = $query->executeQuery();
  381. $hasReservedJobs = $result->fetch() !== false;
  382. $result->closeCursor();
  383. return $hasReservedJobs;
  384. } catch (Exception $e) {
  385. $this->logger->debug('Querying reserved jobs failed', ['exception' => $e]);
  386. return false;
  387. }
  388. }
  389. public function countByClass(): array {
  390. $query = $this->connection->getQueryBuilder();
  391. $query->select('class')
  392. ->selectAlias($query->func()->count('id'), 'count')
  393. ->from('jobs')
  394. ->orderBy('count')
  395. ->groupBy('class');
  396. $result = $query->executeQuery();
  397. $jobs = [];
  398. while (($row = $result->fetch()) !== false) {
  399. /**
  400. * @var array{count:int, class:class-string} $row
  401. */
  402. $jobs[] = $row;
  403. }
  404. return $jobs;
  405. }
  406. }