1
0

Scanner.php 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304
  1. <?php
  2. /**
  3. * @copyright Copyright (c) 2016, ownCloud, Inc.
  4. *
  5. * @author Blaok <i@blaok.me>
  6. * @author Christoph Wurst <christoph@winzerhof-wurst.at>
  7. * @author Jörn Friedrich Dreyer <jfd@butonic.de>
  8. * @author Morris Jobke <hey@morrisjobke.de>
  9. * @author Olivier Paroz <github@oparoz.com>
  10. * @author Robin Appelman <robin@icewind.nl>
  11. * @author Roeland Jago Douma <roeland@famdouma.nl>
  12. * @author Thomas Müller <thomas.mueller@tmit.eu>
  13. * @author Vincent Petry <vincent@nextcloud.com>
  14. *
  15. * @license AGPL-3.0
  16. *
  17. * This code is free software: you can redistribute it and/or modify
  18. * it under the terms of the GNU Affero General Public License, version 3,
  19. * as published by the Free Software Foundation.
  20. *
  21. * This program is distributed in the hope that it will be useful,
  22. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  23. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  24. * GNU Affero General Public License for more details.
  25. *
  26. * You should have received a copy of the GNU Affero General Public License, version 3,
  27. * along with this program. If not, see <http://www.gnu.org/licenses/>
  28. *
  29. */
  30. namespace OC\Files\Utils;
  31. use OC\Files\Cache\Cache;
  32. use OC\Files\Filesystem;
  33. use OC\Files\Storage\FailedStorage;
  34. use OC\Files\Storage\Home;
  35. use OC\ForbiddenException;
  36. use OC\Hooks\PublicEmitter;
  37. use OC\Lock\DBLockingProvider;
  38. use OCA\Files_Sharing\SharedStorage;
  39. use OCP\EventDispatcher\IEventDispatcher;
  40. use OCP\Files\Events\BeforeFileScannedEvent;
  41. use OCP\Files\Events\BeforeFolderScannedEvent;
  42. use OCP\Files\Events\NodeAddedToCache;
  43. use OCP\Files\Events\FileCacheUpdated;
  44. use OCP\Files\Events\NodeRemovedFromCache;
  45. use OCP\Files\Events\FileScannedEvent;
  46. use OCP\Files\Events\FolderScannedEvent;
  47. use OCP\Files\NotFoundException;
  48. use OCP\Files\Storage\IStorage;
  49. use OCP\Files\StorageNotAvailableException;
  50. use OCP\IDBConnection;
  51. use Psr\Log\LoggerInterface;
  52. /**
  53. * Class Scanner
  54. *
  55. * Hooks available in scope \OC\Utils\Scanner
  56. * - scanFile(string $absolutePath)
  57. * - scanFolder(string $absolutePath)
  58. *
  59. * @package OC\Files\Utils
  60. */
  61. class Scanner extends PublicEmitter {
  62. public const MAX_ENTRIES_TO_COMMIT = 10000;
  63. /** @var string $user */
  64. private $user;
  65. /** @var IDBConnection */
  66. protected $db;
  67. /** @var IEventDispatcher */
  68. private $dispatcher;
  69. protected LoggerInterface $logger;
  70. /**
  71. * Whether to use a DB transaction
  72. *
  73. * @var bool
  74. */
  75. protected $useTransaction;
  76. /**
  77. * Number of entries scanned to commit
  78. *
  79. * @var int
  80. */
  81. protected $entriesToCommit;
  82. /**
  83. * @param string $user
  84. * @param IDBConnection|null $db
  85. * @param IEventDispatcher $dispatcher
  86. */
  87. public function __construct($user, $db, IEventDispatcher $dispatcher, LoggerInterface $logger) {
  88. $this->user = $user;
  89. $this->db = $db;
  90. $this->dispatcher = $dispatcher;
  91. $this->logger = $logger;
  92. // when DB locking is used, no DB transactions will be used
  93. $this->useTransaction = !(\OC::$server->getLockingProvider() instanceof DBLockingProvider);
  94. }
  95. /**
  96. * get all storages for $dir
  97. *
  98. * @param string $dir
  99. * @return \OC\Files\Mount\MountPoint[]
  100. */
  101. protected function getMounts($dir) {
  102. //TODO: move to the node based fileapi once that's done
  103. \OC_Util::tearDownFS();
  104. \OC_Util::setupFS($this->user);
  105. $mountManager = Filesystem::getMountManager();
  106. $mounts = $mountManager->findIn($dir);
  107. $mounts[] = $mountManager->find($dir);
  108. $mounts = array_reverse($mounts); //start with the mount of $dir
  109. return $mounts;
  110. }
  111. /**
  112. * attach listeners to the scanner
  113. *
  114. * @param \OC\Files\Mount\MountPoint $mount
  115. */
  116. protected function attachListener($mount) {
  117. $scanner = $mount->getStorage()->getScanner();
  118. $scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function ($path) use ($mount) {
  119. $this->emit('\OC\Files\Utils\Scanner', 'scanFile', [$mount->getMountPoint() . $path]);
  120. $this->dispatcher->dispatchTyped(new BeforeFileScannedEvent($mount->getMountPoint() . $path));
  121. });
  122. $scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount) {
  123. $this->emit('\OC\Files\Utils\Scanner', 'scanFolder', [$mount->getMountPoint() . $path]);
  124. $this->dispatcher->dispatchTyped(new BeforeFolderScannedEvent($mount->getMountPoint() . $path));
  125. });
  126. $scanner->listen('\OC\Files\Cache\Scanner', 'postScanFile', function ($path) use ($mount) {
  127. $this->emit('\OC\Files\Utils\Scanner', 'postScanFile', [$mount->getMountPoint() . $path]);
  128. $this->dispatcher->dispatchTyped(new FileScannedEvent($mount->getMountPoint() . $path));
  129. });
  130. $scanner->listen('\OC\Files\Cache\Scanner', 'postScanFolder', function ($path) use ($mount) {
  131. $this->emit('\OC\Files\Utils\Scanner', 'postScanFolder', [$mount->getMountPoint() . $path]);
  132. $this->dispatcher->dispatchTyped(new FolderScannedEvent($mount->getMountPoint() . $path));
  133. });
  134. $scanner->listen('\OC\Files\Cache\Scanner', 'normalizedNameMismatch', function ($path) use ($mount) {
  135. $this->emit('\OC\Files\Utils\Scanner', 'normalizedNameMismatch', [$path]);
  136. });
  137. }
  138. /**
  139. * @param string $dir
  140. */
  141. public function backgroundScan($dir) {
  142. $mounts = $this->getMounts($dir);
  143. foreach ($mounts as $mount) {
  144. $storage = $mount->getStorage();
  145. if (is_null($storage)) {
  146. continue;
  147. }
  148. // don't bother scanning failed storages (shortcut for same result)
  149. if ($storage->instanceOfStorage(FailedStorage::class)) {
  150. continue;
  151. }
  152. $scanner = $storage->getScanner();
  153. $this->attachListener($mount);
  154. $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) {
  155. $this->triggerPropagator($storage, $path);
  156. });
  157. $scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) {
  158. $this->triggerPropagator($storage, $path);
  159. });
  160. $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) {
  161. $this->triggerPropagator($storage, $path);
  162. });
  163. $propagator = $storage->getPropagator();
  164. $propagator->beginBatch();
  165. $scanner->backgroundScan();
  166. $propagator->commitBatch();
  167. }
  168. }
  169. /**
  170. * @param string $dir
  171. * @param $recursive
  172. * @param callable|null $mountFilter
  173. * @throws ForbiddenException
  174. * @throws NotFoundException
  175. */
  176. public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECURSIVE, callable $mountFilter = null) {
  177. if (!Filesystem::isValidPath($dir)) {
  178. throw new \InvalidArgumentException('Invalid path to scan');
  179. }
  180. $mounts = $this->getMounts($dir);
  181. foreach ($mounts as $mount) {
  182. if ($mountFilter && !$mountFilter($mount)) {
  183. continue;
  184. }
  185. $storage = $mount->getStorage();
  186. if (is_null($storage)) {
  187. continue;
  188. }
  189. // don't bother scanning failed storages (shortcut for same result)
  190. if ($storage->instanceOfStorage(FailedStorage::class)) {
  191. continue;
  192. }
  193. // if the home storage isn't writable then the scanner is run as the wrong user
  194. if ($storage->instanceOfStorage(Home::class)) {
  195. /** @var Home $storage */
  196. foreach (['', 'files'] as $path) {
  197. if (!$storage->isCreatable($path)) {
  198. $fullPath = $storage->getSourcePath($path);
  199. if (!$storage->is_dir($path) && $storage->getCache()->inCache($path)) {
  200. throw new NotFoundException("User folder $fullPath exists in cache but not on disk");
  201. } elseif ($storage->is_dir($path)) {
  202. $ownerUid = fileowner($fullPath);
  203. $owner = posix_getpwuid($ownerUid);
  204. $owner = $owner['name'] ?? $ownerUid;
  205. $permissions = decoct(fileperms($fullPath));
  206. throw new ForbiddenException("User folder $fullPath is not writable, folders is owned by $owner and has mode $permissions");
  207. } else {
  208. // if the root exists in neither the cache nor the storage the user isn't setup yet
  209. break 2;
  210. }
  211. }
  212. }
  213. }
  214. // don't scan received local shares, these can be scanned when scanning the owner's storage
  215. if ($storage->instanceOfStorage(SharedStorage::class)) {
  216. continue;
  217. }
  218. $relativePath = $mount->getInternalPath($dir);
  219. $scanner = $storage->getScanner();
  220. $scanner->setUseTransactions(false);
  221. $this->attachListener($mount);
  222. $scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) {
  223. $this->postProcessEntry($storage, $path);
  224. $this->dispatcher->dispatchTyped(new NodeRemovedFromCache($storage, $path));
  225. });
  226. $scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) {
  227. $this->postProcessEntry($storage, $path);
  228. $this->dispatcher->dispatchTyped(new FileCacheUpdated($storage, $path));
  229. });
  230. $scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) {
  231. $this->postProcessEntry($storage, $path);
  232. $this->dispatcher->dispatchTyped(new NodeAddedToCache($storage, $path));
  233. });
  234. if (!$storage->file_exists($relativePath)) {
  235. throw new NotFoundException($dir);
  236. }
  237. if ($this->useTransaction) {
  238. $this->db->beginTransaction();
  239. }
  240. try {
  241. $propagator = $storage->getPropagator();
  242. $propagator->beginBatch();
  243. $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE);
  244. $cache = $storage->getCache();
  245. if ($cache instanceof Cache) {
  246. // only re-calculate for the root folder we scanned, anything below that is taken care of by the scanner
  247. $cache->correctFolderSize($relativePath);
  248. }
  249. $propagator->commitBatch();
  250. } catch (StorageNotAvailableException $e) {
  251. $this->logger->error('Storage ' . $storage->getId() . ' not available', ['exception' => $e]);
  252. $this->emit('\OC\Files\Utils\Scanner', 'StorageNotAvailable', [$e]);
  253. }
  254. if ($this->useTransaction) {
  255. $this->db->commit();
  256. }
  257. }
  258. }
  259. private function triggerPropagator(IStorage $storage, $internalPath) {
  260. $storage->getPropagator()->propagateChange($internalPath, time());
  261. }
  262. private function postProcessEntry(IStorage $storage, $internalPath) {
  263. $this->triggerPropagator($storage, $internalPath);
  264. if ($this->useTransaction) {
  265. $this->entriesToCommit++;
  266. if ($this->entriesToCommit >= self::MAX_ENTRIES_TO_COMMIT) {
  267. $propagator = $storage->getPropagator();
  268. $this->entriesToCommit = 0;
  269. $this->db->commit();
  270. $propagator->commitBatch();
  271. $this->db->beginTransaction();
  272. $propagator->beginBatch();
  273. }
  274. }
  275. }
  276. }