Propagator.php 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216
  1. <?php
  2. /**
  3. * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
  4. * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
  5. * SPDX-License-Identifier: AGPL-3.0-only
  6. */
  7. namespace OC\Files\Cache;
  8. use OC\DB\Exceptions\DbalException;
  9. use OC\Files\Storage\Wrapper\Encryption;
  10. use OCP\DB\QueryBuilder\IQueryBuilder;
  11. use OCP\Files\Cache\IPropagator;
  12. use OCP\Files\Storage\IReliableEtagStorage;
  13. use OCP\IDBConnection;
  14. use Psr\Log\LoggerInterface;
  15. /**
  16. * Propagate etags and mtimes within the storage
  17. */
  18. class Propagator implements IPropagator {
  19. public const MAX_RETRIES = 3;
  20. private $inBatch = false;
  21. private $batch = [];
  22. /**
  23. * @var \OC\Files\Storage\Storage
  24. */
  25. protected $storage;
  26. /**
  27. * @var IDBConnection
  28. */
  29. private $connection;
  30. /**
  31. * @var array
  32. */
  33. private $ignore = [];
  34. public function __construct(\OC\Files\Storage\Storage $storage, IDBConnection $connection, array $ignore = []) {
  35. $this->storage = $storage;
  36. $this->connection = $connection;
  37. $this->ignore = $ignore;
  38. }
  39. /**
  40. * @param string $internalPath
  41. * @param int $time
  42. * @param int $sizeDifference number of bytes the file has grown
  43. */
  44. public function propagateChange($internalPath, $time, $sizeDifference = 0) {
  45. // Do not propagate changes in ignored paths
  46. foreach ($this->ignore as $ignore) {
  47. if (str_starts_with($internalPath, $ignore)) {
  48. return;
  49. }
  50. }
  51. $storageId = (int)$this->storage->getStorageCache()->getNumericId();
  52. $parents = $this->getParents($internalPath);
  53. if ($this->inBatch) {
  54. foreach ($parents as $parent) {
  55. $this->addToBatch($parent, $time, $sizeDifference);
  56. }
  57. return;
  58. }
  59. $parentHashes = array_map('md5', $parents);
  60. $etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag
  61. $builder = $this->connection->getQueryBuilder();
  62. $hashParams = array_map(function ($hash) use ($builder) {
  63. return $builder->expr()->literal($hash);
  64. }, $parentHashes);
  65. $builder->update('filecache')
  66. ->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter((int)$time, IQueryBuilder::PARAM_INT)))
  67. ->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
  68. ->andWhere($builder->expr()->in('path_hash', $hashParams));
  69. if (!$this->storage->instanceOfStorage(IReliableEtagStorage::class)) {
  70. $builder->set('etag', $builder->createNamedParameter($etag, IQueryBuilder::PARAM_STR));
  71. }
  72. if ($sizeDifference !== 0) {
  73. $hasCalculatedSize = $builder->expr()->gt('size', $builder->expr()->literal(-1, IQUeryBuilder::PARAM_INT));
  74. $sizeColumn = $builder->getColumnName('size');
  75. $newSize = $builder->func()->greatest(
  76. $builder->func()->add('size', $builder->createNamedParameter($sizeDifference)),
  77. $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
  78. );
  79. // Only update if row had a previously calculated size
  80. $builder->set('size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newSize ELSE $sizeColumn END"));
  81. if ($this->storage->instanceOfStorage(Encryption::class)) {
  82. // in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size
  83. $hasUnencryptedSize = $builder->expr()->neq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT));
  84. $sizeColumn = $builder->getColumnName('size');
  85. $unencryptedSizeColumn = $builder->getColumnName('unencrypted_size');
  86. $newUnencryptedSize = $builder->func()->greatest(
  87. $builder->func()->add(
  88. $builder->createFunction("CASE WHEN $hasUnencryptedSize THEN $unencryptedSizeColumn ELSE $sizeColumn END"),
  89. $builder->createNamedParameter($sizeDifference)
  90. ),
  91. $builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
  92. );
  93. // Only update if row had a previously calculated size
  94. $builder->set('unencrypted_size', $builder->createFunction("CASE WHEN $hasCalculatedSize THEN $newUnencryptedSize ELSE $unencryptedSizeColumn END"));
  95. }
  96. }
  97. for ($i = 0; $i < self::MAX_RETRIES; $i++) {
  98. try {
  99. $builder->executeStatement();
  100. break;
  101. } catch (DbalException $e) {
  102. if (!$e->isRetryable()) {
  103. throw $e;
  104. }
  105. /** @var LoggerInterface $loggerInterface */
  106. $loggerInterface = \OCP\Server::get(LoggerInterface::class);
  107. $loggerInterface->warning('Retrying propagation query after retryable exception.', [ 'exception' => $e ]);
  108. }
  109. }
  110. }
  111. protected function getParents($path) {
  112. $parts = explode('/', $path);
  113. $parent = '';
  114. $parents = [];
  115. foreach ($parts as $part) {
  116. $parents[] = $parent;
  117. $parent = trim($parent . '/' . $part, '/');
  118. }
  119. return $parents;
  120. }
  121. /**
  122. * Mark the beginning of a propagation batch
  123. *
  124. * Note that not all cache setups support propagation in which case this will be a noop
  125. *
  126. * Batching for cache setups that do support it has to be explicit since the cache state is not fully consistent
  127. * before the batch is committed.
  128. */
  129. public function beginBatch() {
  130. $this->inBatch = true;
  131. }
  132. private function addToBatch($internalPath, $time, $sizeDifference) {
  133. if (!isset($this->batch[$internalPath])) {
  134. $this->batch[$internalPath] = [
  135. 'hash' => md5($internalPath),
  136. 'time' => $time,
  137. 'size' => $sizeDifference,
  138. ];
  139. } else {
  140. $this->batch[$internalPath]['size'] += $sizeDifference;
  141. if ($time > $this->batch[$internalPath]['time']) {
  142. $this->batch[$internalPath]['time'] = $time;
  143. }
  144. }
  145. }
  146. /**
  147. * Commit the active propagation batch
  148. */
  149. public function commitBatch() {
  150. if (!$this->inBatch) {
  151. throw new \BadMethodCallException('Not in batch');
  152. }
  153. $this->inBatch = false;
  154. $this->connection->beginTransaction();
  155. $query = $this->connection->getQueryBuilder();
  156. $storageId = (int)$this->storage->getStorageCache()->getNumericId();
  157. $query->update('filecache')
  158. ->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
  159. ->set('etag', $query->expr()->literal(uniqid()))
  160. ->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
  161. ->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));
  162. $sizeQuery = $this->connection->getQueryBuilder();
  163. $sizeQuery->update('filecache')
  164. ->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size')))
  165. ->where($query->expr()->eq('storage', $sizeQuery->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
  166. ->andWhere($query->expr()->eq('path_hash', $sizeQuery->createParameter('hash')))
  167. ->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->createNamedParameter(-1, IQueryBuilder::PARAM_INT)));
  168. foreach ($this->batch as $item) {
  169. $query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT);
  170. $query->setParameter('hash', $item['hash']);
  171. $query->executeStatement();
  172. if ($item['size']) {
  173. $sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT);
  174. $sizeQuery->setParameter('hash', $item['hash']);
  175. $sizeQuery->executeStatement();
  176. }
  177. }
  178. $this->batch = [];
  179. $this->connection->commit();
  180. }
  181. }