JobList.php 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419
  1. <?php
  2. /**
  3. * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
  4. * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
  5. * SPDX-License-Identifier: AGPL-3.0-only
  6. */
  7. namespace OC\BackgroundJob;
  8. use OCP\AppFramework\QueryException;
  9. use OCP\AppFramework\Utility\ITimeFactory;
  10. use OCP\AutoloadNotAllowedException;
  11. use OCP\BackgroundJob\IJob;
  12. use OCP\BackgroundJob\IJobList;
  13. use OCP\BackgroundJob\IParallelAwareJob;
  14. use OCP\DB\Exception;
  15. use OCP\DB\QueryBuilder\IQueryBuilder;
  16. use OCP\IConfig;
  17. use OCP\IDBConnection;
  18. use Psr\Log\LoggerInterface;
  19. use function get_class;
  20. use function json_encode;
  21. use function strlen;
  22. class JobList implements IJobList {
  23. public function __construct(
  24. protected IDBConnection $connection,
  25. protected IConfig $config,
  26. protected ITimeFactory $timeFactory,
  27. protected LoggerInterface $logger
  28. ) {
  29. }
  30. public function add($job, $argument = null, ?int $firstCheck = null): void {
  31. if ($firstCheck === null) {
  32. $firstCheck = $this->timeFactory->getTime();
  33. }
  34. $class = ($job instanceof IJob) ? get_class($job) : $job;
  35. $argumentJson = json_encode($argument);
  36. if (strlen($argumentJson) > 4000) {
  37. throw new \InvalidArgumentException('Background job arguments can\'t exceed 4000 characters (json encoded)');
  38. }
  39. $query = $this->connection->getQueryBuilder();
  40. if (!$this->has($job, $argument)) {
  41. $query->insert('jobs')
  42. ->values([
  43. 'class' => $query->createNamedParameter($class),
  44. 'argument' => $query->createNamedParameter($argumentJson),
  45. 'argument_hash' => $query->createNamedParameter(hash('sha256', $argumentJson)),
  46. 'last_run' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
  47. 'last_checked' => $query->createNamedParameter($firstCheck, IQueryBuilder::PARAM_INT),
  48. ]);
  49. } else {
  50. $query->update('jobs')
  51. ->set('reserved_at', $query->expr()->literal(0, IQueryBuilder::PARAM_INT))
  52. ->set('last_checked', $query->createNamedParameter($firstCheck, IQueryBuilder::PARAM_INT))
  53. ->set('last_run', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))
  54. ->where($query->expr()->eq('class', $query->createNamedParameter($class)))
  55. ->andWhere($query->expr()->eq('argument_hash', $query->createNamedParameter(hash('sha256', $argumentJson))));
  56. }
  57. $query->executeStatement();
  58. }
  59. public function scheduleAfter(string $job, int $runAfter, $argument = null): void {
  60. $this->add($job, $argument, $runAfter);
  61. }
  62. /**
  63. * @param IJob|string $job
  64. * @param mixed $argument
  65. */
  66. public function remove($job, $argument = null): void {
  67. $class = ($job instanceof IJob) ? get_class($job) : $job;
  68. $query = $this->connection->getQueryBuilder();
  69. $query->delete('jobs')
  70. ->where($query->expr()->eq('class', $query->createNamedParameter($class)));
  71. if (!is_null($argument)) {
  72. $argumentJson = json_encode($argument);
  73. $query->andWhere($query->expr()->eq('argument_hash', $query->createNamedParameter(hash('sha256', $argumentJson))));
  74. }
  75. // Add galera safe delete chunking if using mysql
  76. // Stops us hitting wsrep_max_ws_rows when large row counts are deleted
  77. if ($this->connection->getDatabaseProvider() === IDBConnection::PLATFORM_MYSQL) {
  78. // Then use chunked delete
  79. $max = IQueryBuilder::MAX_ROW_DELETION;
  80. $query->setMaxResults($max);
  81. do {
  82. $deleted = $query->executeStatement();
  83. } while ($deleted === $max);
  84. } else {
  85. // Dont use chunked delete - let the DB handle the large row count natively
  86. $query->executeStatement();
  87. }
  88. }
  89. public function removeById(int $id): void {
  90. $query = $this->connection->getQueryBuilder();
  91. $query->delete('jobs')
  92. ->where($query->expr()->eq('id', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
  93. $query->executeStatement();
  94. }
  95. /**
  96. * check if a job is in the list
  97. *
  98. * @param IJob|class-string<IJob> $job
  99. * @param mixed $argument
  100. */
  101. public function has($job, $argument): bool {
  102. $class = ($job instanceof IJob) ? get_class($job) : $job;
  103. $argument = json_encode($argument);
  104. $query = $this->connection->getQueryBuilder();
  105. $query->select('id')
  106. ->from('jobs')
  107. ->where($query->expr()->eq('class', $query->createNamedParameter($class)))
  108. ->andWhere($query->expr()->eq('argument_hash', $query->createNamedParameter(hash('sha256', $argument))))
  109. ->setMaxResults(1);
  110. $result = $query->executeQuery();
  111. $row = $result->fetch();
  112. $result->closeCursor();
  113. return (bool) $row;
  114. }
  115. public function getJobs($job, ?int $limit, int $offset): array {
  116. $iterable = $this->getJobsIterator($job, $limit, $offset);
  117. return (is_array($iterable))
  118. ? $iterable
  119. : iterator_to_array($iterable);
  120. }
  121. /**
  122. * @param IJob|class-string<IJob>|null $job
  123. * @return iterable<IJob> Avoid to store these objects as they may share a Singleton instance. You should instead use these IJobs instances while looping on the iterable.
  124. */
  125. public function getJobsIterator($job, ?int $limit, int $offset): iterable {
  126. $query = $this->connection->getQueryBuilder();
  127. $query->select('*')
  128. ->from('jobs')
  129. ->setMaxResults($limit)
  130. ->setFirstResult($offset);
  131. if ($job !== null) {
  132. $class = ($job instanceof IJob) ? get_class($job) : $job;
  133. $query->where($query->expr()->eq('class', $query->createNamedParameter($class)));
  134. }
  135. $result = $query->executeQuery();
  136. while ($row = $result->fetch()) {
  137. $job = $this->buildJob($row);
  138. if ($job) {
  139. yield $job;
  140. }
  141. }
  142. $result->closeCursor();
  143. }
  144. /**
  145. * @inheritDoc
  146. */
  147. public function getNext(bool $onlyTimeSensitive = false, ?array $jobClasses = null): ?IJob {
  148. $query = $this->connection->getQueryBuilder();
  149. $query->select('*')
  150. ->from('jobs')
  151. ->where($query->expr()->lte('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - 12 * 3600, IQueryBuilder::PARAM_INT)))
  152. ->andWhere($query->expr()->lte('last_checked', $query->createNamedParameter($this->timeFactory->getTime(), IQueryBuilder::PARAM_INT)))
  153. ->orderBy('last_checked', 'ASC')
  154. ->setMaxResults(1);
  155. if ($onlyTimeSensitive) {
  156. $query->andWhere($query->expr()->eq('time_sensitive', $query->createNamedParameter(IJob::TIME_SENSITIVE, IQueryBuilder::PARAM_INT)));
  157. }
  158. if (!empty($jobClasses)) {
  159. $orClasses = [];
  160. foreach ($jobClasses as $jobClass) {
  161. $orClasses[] = $query->expr()->eq('class', $query->createNamedParameter($jobClass, IQueryBuilder::PARAM_STR));
  162. }
  163. $query->andWhere($query->expr()->orX(...$orClasses));
  164. }
  165. $result = $query->executeQuery();
  166. $row = $result->fetch();
  167. $result->closeCursor();
  168. if ($row) {
  169. $job = $this->buildJob($row);
  170. if ($job instanceof IParallelAwareJob && !$job->getAllowParallelRuns() && $this->hasReservedJob(get_class($job))) {
  171. $this->logger->info('Skipping ' . get_class($job) . ' job with ID ' . $job->getId() . ' because another job with the same class is already running', ['app' => 'cron']);
  172. $update = $this->connection->getQueryBuilder();
  173. $update->update('jobs')
  174. ->set('last_checked', $update->createNamedParameter($this->timeFactory->getTime() + 1))
  175. ->where($update->expr()->eq('id', $update->createParameter('jobid')));
  176. $update->setParameter('jobid', $row['id']);
  177. $update->executeStatement();
  178. return $this->getNext($onlyTimeSensitive);
  179. }
  180. $update = $this->connection->getQueryBuilder();
  181. $update->update('jobs')
  182. ->set('reserved_at', $update->createNamedParameter($this->timeFactory->getTime()))
  183. ->set('last_checked', $update->createNamedParameter($this->timeFactory->getTime()))
  184. ->where($update->expr()->eq('id', $update->createParameter('jobid')))
  185. ->andWhere($update->expr()->eq('reserved_at', $update->createParameter('reserved_at')))
  186. ->andWhere($update->expr()->eq('last_checked', $update->createParameter('last_checked')));
  187. $update->setParameter('jobid', $row['id']);
  188. $update->setParameter('reserved_at', $row['reserved_at']);
  189. $update->setParameter('last_checked', $row['last_checked']);
  190. $count = $update->executeStatement();
  191. if ($count === 0) {
  192. // Background job already executed elsewhere, try again.
  193. return $this->getNext($onlyTimeSensitive, $jobClasses);
  194. }
  195. if ($job === null) {
  196. // set the last_checked to 12h in the future to not check failing jobs all over again
  197. $reset = $this->connection->getQueryBuilder();
  198. $reset->update('jobs')
  199. ->set('reserved_at', $reset->expr()->literal(0, IQueryBuilder::PARAM_INT))
  200. ->set('last_checked', $reset->createNamedParameter($this->timeFactory->getTime() + 12 * 3600, IQueryBuilder::PARAM_INT))
  201. ->where($reset->expr()->eq('id', $reset->createNamedParameter($row['id'], IQueryBuilder::PARAM_INT)));
  202. $reset->executeStatement();
  203. // Background job from disabled app, try again.
  204. return $this->getNext($onlyTimeSensitive, $jobClasses);
  205. }
  206. return $job;
  207. } else {
  208. return null;
  209. }
  210. }
  211. /**
  212. * @return ?IJob The job matching the id. Beware that this object may be a singleton and may be modified by the next call to buildJob.
  213. */
  214. public function getById(int $id): ?IJob {
  215. $row = $this->getDetailsById($id);
  216. if ($row) {
  217. return $this->buildJob($row);
  218. }
  219. return null;
  220. }
  221. public function getDetailsById(int $id): ?array {
  222. $query = $this->connection->getQueryBuilder();
  223. $query->select('*')
  224. ->from('jobs')
  225. ->where($query->expr()->eq('id', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
  226. $result = $query->executeQuery();
  227. $row = $result->fetch();
  228. $result->closeCursor();
  229. if ($row) {
  230. return $row;
  231. }
  232. return null;
  233. }
  234. /**
  235. * get the job object from a row in the db
  236. *
  237. * @param array{class:class-string<IJob>, id:mixed, last_run:mixed, argument:string} $row
  238. * @return ?IJob the next job to run. Beware that this object may be a singleton and may be modified by the next call to buildJob.
  239. */
  240. private function buildJob(array $row): ?IJob {
  241. try {
  242. try {
  243. // Try to load the job as a service
  244. /** @var IJob $job */
  245. $job = \OCP\Server::get($row['class']);
  246. } catch (QueryException $e) {
  247. if (class_exists($row['class'])) {
  248. $class = $row['class'];
  249. $job = new $class();
  250. } else {
  251. // Remove job from disabled app or old version of an app
  252. $this->removeById($row['id']);
  253. return null;
  254. }
  255. }
  256. if (!($job instanceof IJob)) {
  257. // This most likely means an invalid job was enqueued. We can ignore it.
  258. return null;
  259. }
  260. $job->setId((int) $row['id']);
  261. $job->setLastRun((int) $row['last_run']);
  262. $job->setArgument(json_decode($row['argument'], true));
  263. return $job;
  264. } catch (AutoloadNotAllowedException $e) {
  265. // job is from a disabled app, ignore
  266. return null;
  267. }
  268. }
  269. /**
  270. * set the job that was last ran
  271. */
  272. public function setLastJob(IJob $job): void {
  273. $this->unlockJob($job);
  274. $this->config->setAppValue('backgroundjob', 'lastjob', (string)$job->getId());
  275. }
  276. /**
  277. * Remove the reservation for a job
  278. */
  279. public function unlockJob(IJob $job): void {
  280. $query = $this->connection->getQueryBuilder();
  281. $query->update('jobs')
  282. ->set('reserved_at', $query->expr()->literal(0, IQueryBuilder::PARAM_INT))
  283. ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId(), IQueryBuilder::PARAM_INT)));
  284. $query->executeStatement();
  285. }
  286. /**
  287. * set the lastRun of $job to now
  288. */
  289. public function setLastRun(IJob $job): void {
  290. $query = $this->connection->getQueryBuilder();
  291. $query->update('jobs')
  292. ->set('last_run', $query->createNamedParameter(time(), IQueryBuilder::PARAM_INT))
  293. ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId(), IQueryBuilder::PARAM_INT)));
  294. if ($job instanceof \OCP\BackgroundJob\TimedJob
  295. && !$job->isTimeSensitive()) {
  296. $query->set('time_sensitive', $query->createNamedParameter(IJob::TIME_INSENSITIVE));
  297. }
  298. $query->executeStatement();
  299. }
  300. /**
  301. * @param int $timeTaken
  302. */
  303. public function setExecutionTime(IJob $job, $timeTaken): void {
  304. $query = $this->connection->getQueryBuilder();
  305. $query->update('jobs')
  306. ->set('execution_duration', $query->createNamedParameter($timeTaken, IQueryBuilder::PARAM_INT))
  307. ->set('reserved_at', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))
  308. ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId(), IQueryBuilder::PARAM_INT)));
  309. $query->executeStatement();
  310. }
  311. /**
  312. * Reset the $job so it executes on the next trigger
  313. *
  314. * @since 23.0.0
  315. */
  316. public function resetBackgroundJob(IJob $job): void {
  317. $query = $this->connection->getQueryBuilder();
  318. $query->update('jobs')
  319. ->set('last_run', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))
  320. ->set('reserved_at', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))
  321. ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId()), IQueryBuilder::PARAM_INT));
  322. $query->executeStatement();
  323. }
  324. public function hasReservedJob(?string $className = null): bool {
  325. $query = $this->connection->getQueryBuilder();
  326. $query->select('*')
  327. ->from('jobs')
  328. ->where($query->expr()->gt('reserved_at', $query->createNamedParameter($this->timeFactory->getTime() - 6 * 3600, IQueryBuilder::PARAM_INT)))
  329. ->setMaxResults(1);
  330. if ($className !== null) {
  331. $query->andWhere($query->expr()->eq('class', $query->createNamedParameter($className)));
  332. }
  333. try {
  334. $result = $query->executeQuery();
  335. $hasReservedJobs = $result->fetch() !== false;
  336. $result->closeCursor();
  337. return $hasReservedJobs;
  338. } catch (Exception $e) {
  339. $this->logger->debug('Querying reserved jobs failed', ['exception' => $e]);
  340. return false;
  341. }
  342. }
  343. public function countByClass(): array {
  344. $query = $this->connection->getQueryBuilder();
  345. $query->select('class')
  346. ->selectAlias($query->func()->count('id'), 'count')
  347. ->from('jobs')
  348. ->orderBy('count')
  349. ->groupBy('class');
  350. $result = $query->executeQuery();
  351. $jobs = [];
  352. while (($row = $result->fetch()) !== false) {
  353. /**
  354. * @var array{count:int, class:class-string} $row
  355. */
  356. $jobs[] = $row;
  357. }
  358. return $jobs;
  359. }
  360. }