1
0

DeleteOrphanedFiles.php 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161
  1. <?php
  2. /**
  3. * SPDX-FileCopyrightText: 2017-2024 Nextcloud GmbH and Nextcloud contributors
  4. * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
  5. * SPDX-License-Identifier: AGPL-3.0-only
  6. */
  7. namespace OCA\Files\Command;
  8. use OCP\DB\QueryBuilder\IQueryBuilder;
  9. use OCP\IDBConnection;
  10. use Symfony\Component\Console\Command\Command;
  11. use Symfony\Component\Console\Input\InputInterface;
  12. use Symfony\Component\Console\Input\InputOption;
  13. use Symfony\Component\Console\Output\OutputInterface;
  14. /**
  15. * Delete all file entries that have no matching entries in the storage table.
  16. */
  17. class DeleteOrphanedFiles extends Command {
  18. public const CHUNK_SIZE = 200;
  19. public function __construct(
  20. protected IDBConnection $connection,
  21. ) {
  22. parent::__construct();
  23. }
  24. protected function configure(): void {
  25. $this
  26. ->setName('files:cleanup')
  27. ->setDescription('cleanup filecache')
  28. ->addOption('skip-filecache-extended', null, InputOption::VALUE_NONE, 'don\'t remove orphaned entries from filecache_extended');
  29. }
  30. public function execute(InputInterface $input, OutputInterface $output): int {
  31. $deletedEntries = 0;
  32. $fileIdsByStorage = [];
  33. $deletedStorages = array_diff($this->getReferencedStorages(), $this->getExistingStorages());
  34. $deleteExtended = !$input->getOption('skip-filecache-extended');
  35. if ($deleteExtended) {
  36. $fileIdsByStorage = $this->getFileIdsForStorages($deletedStorages);
  37. }
  38. $deleteQuery = $this->connection->getQueryBuilder();
  39. $deleteQuery->delete('filecache')
  40. ->where($deleteQuery->expr()->in('storage', $deleteQuery->createParameter('storage_ids')));
  41. $deletedStorageChunks = array_chunk($deletedStorages, self::CHUNK_SIZE);
  42. foreach ($deletedStorageChunks as $deletedStorageChunk) {
  43. $deleteQuery->setParameter('storage_ids', $deletedStorageChunk, IQueryBuilder::PARAM_INT_ARRAY);
  44. $deletedEntries += $deleteQuery->executeStatement();
  45. }
  46. $output->writeln("$deletedEntries orphaned file cache entries deleted");
  47. if ($deleteExtended) {
  48. $deletedFileCacheExtended = $this->cleanupOrphanedFileCacheExtended($fileIdsByStorage);
  49. $output->writeln("$deletedFileCacheExtended orphaned file cache extended entries deleted");
  50. }
  51. $deletedMounts = $this->cleanupOrphanedMounts();
  52. $output->writeln("$deletedMounts orphaned mount entries deleted");
  53. return self::SUCCESS;
  54. }
  55. private function getReferencedStorages(): array {
  56. $query = $this->connection->getQueryBuilder();
  57. $query->select('storage')
  58. ->from('filecache')
  59. ->groupBy('storage')
  60. ->runAcrossAllShards();
  61. return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
  62. }
  63. private function getExistingStorages(): array {
  64. $query = $this->connection->getQueryBuilder();
  65. $query->select('numeric_id')
  66. ->from('storages')
  67. ->groupBy('numeric_id');
  68. return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
  69. }
  70. /**
  71. * @param int[] $storageIds
  72. * @return array<int, int[]>
  73. */
  74. private function getFileIdsForStorages(array $storageIds): array {
  75. $query = $this->connection->getQueryBuilder();
  76. $query->select('storage', 'fileid')
  77. ->from('filecache')
  78. ->where($query->expr()->in('storage', $query->createParameter('storage_ids')));
  79. $result = [];
  80. $storageIdChunks = array_chunk($storageIds, self::CHUNK_SIZE);
  81. foreach ($storageIdChunks as $storageIdChunk) {
  82. $query->setParameter('storage_ids', $storageIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
  83. $chunk = $query->executeQuery()->fetchAll();
  84. foreach ($chunk as $row) {
  85. $result[$row['storage']][] = $row['fileid'];
  86. }
  87. }
  88. return $result;
  89. }
  90. /**
  91. * @param array<int, int[]> $fileIdsByStorage
  92. * @return int
  93. */
  94. private function cleanupOrphanedFileCacheExtended(array $fileIdsByStorage): int {
  95. $deletedEntries = 0;
  96. $deleteQuery = $this->connection->getQueryBuilder();
  97. $deleteQuery->delete('filecache_extended')
  98. ->where($deleteQuery->expr()->in('fileid', $deleteQuery->createParameter('file_ids')));
  99. foreach ($fileIdsByStorage as $storageId => $fileIds) {
  100. $deleteQuery->hintShardKey('storage', $storageId, true);
  101. $fileChunks = array_chunk($fileIds, self::CHUNK_SIZE);
  102. foreach ($fileChunks as $fileChunk) {
  103. $deleteQuery->setParameter('file_ids', $fileChunk, IQueryBuilder::PARAM_INT_ARRAY);
  104. $deletedEntries += $deleteQuery->executeStatement();
  105. }
  106. }
  107. return $deletedEntries;
  108. }
  109. private function cleanupOrphanedMounts(): int {
  110. $deletedEntries = 0;
  111. $query = $this->connection->getQueryBuilder();
  112. $query->select('m.storage_id')
  113. ->from('mounts', 'm')
  114. ->where($query->expr()->isNull('s.numeric_id'))
  115. ->leftJoin('m', 'storages', 's', $query->expr()->eq('m.storage_id', 's.numeric_id'))
  116. ->groupBy('storage_id')
  117. ->setMaxResults(self::CHUNK_SIZE);
  118. $deleteQuery = $this->connection->getQueryBuilder();
  119. $deleteQuery->delete('mounts')
  120. ->where($deleteQuery->expr()->eq('storage_id', $deleteQuery->createParameter('storageid')));
  121. $deletedInLastChunk = self::CHUNK_SIZE;
  122. while ($deletedInLastChunk === self::CHUNK_SIZE) {
  123. $deletedInLastChunk = 0;
  124. $result = $query->execute();
  125. while ($row = $result->fetch()) {
  126. $deletedInLastChunk++;
  127. $deletedEntries += $deleteQuery->setParameter('storageid', (int) $row['storage_id'])
  128. ->execute();
  129. }
  130. $result->closeCursor();
  131. }
  132. return $deletedEntries;
  133. }
  134. }