1
0

Cache.php 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195
  1. <?php
  2. /**
  3. * @copyright Copyright (c) 2016, ownCloud, Inc.
  4. *
  5. * @author Andreas Fischer <bantu@owncloud.com>
  6. * @author Ari Selseng <ari@selseng.net>
  7. * @author Artem Kochnev <MrJeos@gmail.com>
  8. * @author Björn Schießle <bjoern@schiessle.org>
  9. * @author Christoph Wurst <christoph@winzerhof-wurst.at>
  10. * @author Daniel Kesselberg <mail@danielkesselberg.de>
  11. * @author Florin Peter <github@florin-peter.de>
  12. * @author Frédéric Fortier <frederic.fortier@oronospolytechnique.com>
  13. * @author Jens-Christian Fischer <jens-christian.fischer@switch.ch>
  14. * @author Joas Schilling <coding@schilljs.com>
  15. * @author John Molakvoæ <skjnldsv@protonmail.com>
  16. * @author Jörn Friedrich Dreyer <jfd@butonic.de>
  17. * @author Lukas Reschke <lukas@statuscode.ch>
  18. * @author Michael Gapczynski <GapczynskiM@gmail.com>
  19. * @author Morris Jobke <hey@morrisjobke.de>
  20. * @author Robin Appelman <robin@icewind.nl>
  21. * @author Robin McCorkell <robin@mccorkell.me.uk>
  22. * @author Roeland Jago Douma <roeland@famdouma.nl>
  23. * @author Vincent Petry <vincent@nextcloud.com>
  24. *
  25. * @license AGPL-3.0
  26. *
  27. * This code is free software: you can redistribute it and/or modify
  28. * it under the terms of the GNU Affero General Public License, version 3,
  29. * as published by the Free Software Foundation.
  30. *
  31. * This program is distributed in the hope that it will be useful,
  32. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  33. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  34. * GNU Affero General Public License for more details.
  35. *
  36. * You should have received a copy of the GNU Affero General Public License, version 3,
  37. * along with this program. If not, see <http://www.gnu.org/licenses/>
  38. *
  39. */
  40. namespace OC\Files\Cache;
  41. use Doctrine\DBAL\Exception\RetryableException;
  42. use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
  43. use OC\Files\Search\SearchComparison;
  44. use OC\Files\Search\SearchQuery;
  45. use OC\Files\Storage\Wrapper\Encryption;
  46. use OC\SystemConfig;
  47. use OCP\DB\QueryBuilder\IQueryBuilder;
  48. use OCP\EventDispatcher\IEventDispatcher;
  49. use OCP\Files\Cache\CacheEntryInsertedEvent;
  50. use OCP\Files\Cache\CacheEntryRemovedEvent;
  51. use OCP\Files\Cache\CacheEntryUpdatedEvent;
  52. use OCP\Files\Cache\CacheInsertEvent;
  53. use OCP\Files\Cache\CacheUpdateEvent;
  54. use OCP\Files\Cache\ICache;
  55. use OCP\Files\Cache\ICacheEntry;
  56. use OCP\Files\FileInfo;
  57. use OCP\Files\IMimeTypeLoader;
  58. use OCP\Files\Search\ISearchComparison;
  59. use OCP\Files\Search\ISearchOperator;
  60. use OCP\Files\Search\ISearchQuery;
  61. use OCP\Files\Storage\IStorage;
  62. use OCP\FilesMetadata\IFilesMetadataManager;
  63. use OCP\IDBConnection;
  64. use OCP\Util;
  65. use Psr\Log\LoggerInterface;
  66. /**
  67. * Metadata cache for a storage
  68. *
  69. * The cache stores the metadata for all files and folders in a storage and is kept up to date through the following mechanisms:
  70. *
  71. * - Scanner: scans the storage and updates the cache where needed
  72. * - Watcher: checks for changes made to the filesystem outside of the Nextcloud instance and rescans files and folder when a change is detected
  73. * - Updater: listens to changes made to the filesystem inside of the Nextcloud instance and updates the cache where needed
  74. * - ChangePropagator: updates the mtime and etags of parent folders whenever a change to the cache is made to the cache by the updater
  75. */
  76. class Cache implements ICache {
  77. use MoveFromCacheTrait {
  78. MoveFromCacheTrait::moveFromCache as moveFromCacheFallback;
  79. }
  80. /**
  81. * @var array partial data for the cache
  82. */
  83. protected array $partial = [];
  84. protected string $storageId;
  85. protected Storage $storageCache;
  86. protected IMimeTypeLoader$mimetypeLoader;
  87. protected IDBConnection $connection;
  88. protected SystemConfig $systemConfig;
  89. protected LoggerInterface $logger;
  90. protected QuerySearchHelper $querySearchHelper;
  91. protected IEventDispatcher $eventDispatcher;
  92. protected IFilesMetadataManager $metadataManager;
  93. public function __construct(
  94. private IStorage $storage,
  95. // this constructor is used in to many pleases to easily do proper di
  96. // so instead we group it all together
  97. ?CacheDependencies $dependencies = null,
  98. ) {
  99. $this->storageId = $storage->getId();
  100. if (strlen($this->storageId) > 64) {
  101. $this->storageId = md5($this->storageId);
  102. }
  103. if (!$dependencies) {
  104. $dependencies = \OC::$server->get(CacheDependencies::class);
  105. }
  106. $this->storageCache = new Storage($this->storage, true, $dependencies->getConnection());
  107. $this->mimetypeLoader = $dependencies->getMimeTypeLoader();
  108. $this->connection = $dependencies->getConnection();
  109. $this->systemConfig = $dependencies->getSystemConfig();
  110. $this->logger = $dependencies->getLogger();
  111. $this->querySearchHelper = $dependencies->getQuerySearchHelper();
  112. $this->eventDispatcher = $dependencies->getEventDispatcher();
  113. $this->metadataManager = $dependencies->getMetadataManager();
  114. }
  115. protected function getQueryBuilder() {
  116. return new CacheQueryBuilder(
  117. $this->connection,
  118. $this->systemConfig,
  119. $this->logger,
  120. $this->metadataManager,
  121. );
  122. }
  123. public function getStorageCache(): Storage {
  124. return $this->storageCache;
  125. }
  126. /**
  127. * Get the numeric storage id for this cache's storage
  128. *
  129. * @return int
  130. */
  131. public function getNumericStorageId() {
  132. return $this->storageCache->getNumericId();
  133. }
  134. /**
  135. * get the stored metadata of a file or folder
  136. *
  137. * @param string | int $file either the path of a file or folder or the file id for a file or folder
  138. * @return ICacheEntry|false the cache entry as array or false if the file is not found in the cache
  139. */
  140. public function get($file) {
  141. $query = $this->getQueryBuilder();
  142. $query->selectFileCache();
  143. $metadataQuery = $query->selectMetadata();
  144. if (is_string($file) || $file == '') {
  145. // normalize file
  146. $file = $this->normalize($file);
  147. $query->whereStorageId($this->getNumericStorageId())
  148. ->wherePath($file);
  149. } else { //file id
  150. $query->whereFileId($file);
  151. }
  152. $result = $query->execute();
  153. $data = $result->fetch();
  154. $result->closeCursor();
  155. //merge partial data
  156. if (!$data && is_string($file) && isset($this->partial[$file])) {
  157. return $this->partial[$file];
  158. } elseif (!$data) {
  159. return $data;
  160. } else {
  161. $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
  162. return self::cacheEntryFromData($data, $this->mimetypeLoader);
  163. }
  164. }
  165. /**
  166. * Create a CacheEntry from database row
  167. *
  168. * @param array $data
  169. * @param IMimeTypeLoader $mimetypeLoader
  170. * @return CacheEntry
  171. */
  172. public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader) {
  173. //fix types
  174. $data['name'] = (string)$data['name'];
  175. $data['path'] = (string)$data['path'];
  176. $data['fileid'] = (int)$data['fileid'];
  177. $data['parent'] = (int)$data['parent'];
  178. $data['size'] = Util::numericToNumber($data['size']);
  179. $data['unencrypted_size'] = Util::numericToNumber($data['unencrypted_size'] ?? 0);
  180. $data['mtime'] = (int)$data['mtime'];
  181. $data['storage_mtime'] = (int)$data['storage_mtime'];
  182. $data['encryptedVersion'] = (int)$data['encrypted'];
  183. $data['encrypted'] = (bool)$data['encrypted'];
  184. $data['storage_id'] = $data['storage'];
  185. $data['storage'] = (int)$data['storage'];
  186. $data['mimetype'] = $mimetypeLoader->getMimetypeById($data['mimetype']);
  187. $data['mimepart'] = $mimetypeLoader->getMimetypeById($data['mimepart']);
  188. if ($data['storage_mtime'] == 0) {
  189. $data['storage_mtime'] = $data['mtime'];
  190. }
  191. $data['permissions'] = (int)$data['permissions'];
  192. if (isset($data['creation_time'])) {
  193. $data['creation_time'] = (int)$data['creation_time'];
  194. }
  195. if (isset($data['upload_time'])) {
  196. $data['upload_time'] = (int)$data['upload_time'];
  197. }
  198. return new CacheEntry($data);
  199. }
  200. /**
  201. * get the metadata of all files stored in $folder
  202. *
  203. * @param string $folder
  204. * @return ICacheEntry[]
  205. */
  206. public function getFolderContents($folder) {
  207. $fileId = $this->getId($folder);
  208. return $this->getFolderContentsById($fileId);
  209. }
  210. /**
  211. * get the metadata of all files stored in $folder
  212. *
  213. * @param int $fileId the file id of the folder
  214. * @return ICacheEntry[]
  215. */
  216. public function getFolderContentsById($fileId) {
  217. if ($fileId > -1) {
  218. $query = $this->getQueryBuilder();
  219. $query->selectFileCache()
  220. ->whereParent($fileId)
  221. ->orderBy('name', 'ASC');
  222. $metadataQuery = $query->selectMetadata();
  223. $result = $query->execute();
  224. $files = $result->fetchAll();
  225. $result->closeCursor();
  226. return array_map(function (array $data) use ($metadataQuery) {
  227. $data['metadata'] = $metadataQuery->extractMetadata($data)->asArray();
  228. return self::cacheEntryFromData($data, $this->mimetypeLoader);
  229. }, $files);
  230. }
  231. return [];
  232. }
  233. /**
  234. * insert or update meta data for a file or folder
  235. *
  236. * @param string $file
  237. * @param array $data
  238. *
  239. * @return int file id
  240. * @throws \RuntimeException
  241. */
  242. public function put($file, array $data) {
  243. if (($id = $this->getId($file)) > -1) {
  244. $this->update($id, $data);
  245. return $id;
  246. } else {
  247. return $this->insert($file, $data);
  248. }
  249. }
  250. /**
  251. * insert meta data for a new file or folder
  252. *
  253. * @param string $file
  254. * @param array $data
  255. *
  256. * @return int file id
  257. * @throws \RuntimeException
  258. */
  259. public function insert($file, array $data) {
  260. // normalize file
  261. $file = $this->normalize($file);
  262. if (isset($this->partial[$file])) { //add any saved partial data
  263. $data = array_merge($this->partial[$file], $data);
  264. unset($this->partial[$file]);
  265. }
  266. $requiredFields = ['size', 'mtime', 'mimetype'];
  267. foreach ($requiredFields as $field) {
  268. if (!isset($data[$field])) { //data not complete save as partial and return
  269. $this->partial[$file] = $data;
  270. return -1;
  271. }
  272. }
  273. $data['path'] = $file;
  274. if (!isset($data['parent'])) {
  275. $data['parent'] = $this->getParentId($file);
  276. }
  277. $data['name'] = basename($file);
  278. [$values, $extensionValues] = $this->normalizeData($data);
  279. $storageId = $this->getNumericStorageId();
  280. $values['storage'] = $storageId;
  281. try {
  282. $builder = $this->connection->getQueryBuilder();
  283. $builder->insert('filecache');
  284. foreach ($values as $column => $value) {
  285. $builder->setValue($column, $builder->createNamedParameter($value));
  286. }
  287. if ($builder->execute()) {
  288. $fileId = $builder->getLastInsertId();
  289. if (count($extensionValues)) {
  290. $query = $this->getQueryBuilder();
  291. $query->insert('filecache_extended');
  292. $query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT));
  293. foreach ($extensionValues as $column => $value) {
  294. $query->setValue($column, $query->createNamedParameter($value));
  295. }
  296. $query->execute();
  297. }
  298. $event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId);
  299. $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event);
  300. $this->eventDispatcher->dispatchTyped($event);
  301. return $fileId;
  302. }
  303. } catch (UniqueConstraintViolationException $e) {
  304. // entry exists already
  305. if ($this->connection->inTransaction()) {
  306. $this->connection->commit();
  307. $this->connection->beginTransaction();
  308. }
  309. }
  310. // The file was created in the mean time
  311. if (($id = $this->getId($file)) > -1) {
  312. $this->update($id, $data);
  313. return $id;
  314. } else {
  315. throw new \RuntimeException('File entry could not be inserted but could also not be selected with getId() in order to perform an update. Please try again.');
  316. }
  317. }
  318. /**
  319. * update the metadata of an existing file or folder in the cache
  320. *
  321. * @param int $id the fileid of the existing file or folder
  322. * @param array $data [$key => $value] the metadata to update, only the fields provided in the array will be updated, non-provided values will remain unchanged
  323. */
  324. public function update($id, array $data) {
  325. if (isset($data['path'])) {
  326. // normalize path
  327. $data['path'] = $this->normalize($data['path']);
  328. }
  329. if (isset($data['name'])) {
  330. // normalize path
  331. $data['name'] = $this->normalize($data['name']);
  332. }
  333. [$values, $extensionValues] = $this->normalizeData($data);
  334. if (count($values)) {
  335. $query = $this->getQueryBuilder();
  336. $query->update('filecache')
  337. ->whereFileId($id)
  338. ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) {
  339. return $query->expr()->orX(
  340. $query->expr()->neq($key, $query->createNamedParameter($value)),
  341. $query->expr()->isNull($key)
  342. );
  343. }, array_keys($values), array_values($values))));
  344. foreach ($values as $key => $value) {
  345. $query->set($key, $query->createNamedParameter($value));
  346. }
  347. $query->execute();
  348. }
  349. if (count($extensionValues)) {
  350. try {
  351. $query = $this->getQueryBuilder();
  352. $query->insert('filecache_extended');
  353. $query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT));
  354. foreach ($extensionValues as $column => $value) {
  355. $query->setValue($column, $query->createNamedParameter($value));
  356. }
  357. $query->execute();
  358. } catch (UniqueConstraintViolationException $e) {
  359. $query = $this->getQueryBuilder();
  360. $query->update('filecache_extended')
  361. ->whereFileId($id)
  362. ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) {
  363. return $query->expr()->orX(
  364. $query->expr()->neq($key, $query->createNamedParameter($value)),
  365. $query->expr()->isNull($key)
  366. );
  367. }, array_keys($extensionValues), array_values($extensionValues))));
  368. foreach ($extensionValues as $key => $value) {
  369. $query->set($key, $query->createNamedParameter($value));
  370. }
  371. $query->execute();
  372. }
  373. }
  374. $path = $this->getPathById($id);
  375. // path can still be null if the file doesn't exist
  376. if ($path !== null) {
  377. $event = new CacheEntryUpdatedEvent($this->storage, $path, $id, $this->getNumericStorageId());
  378. $this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event);
  379. $this->eventDispatcher->dispatchTyped($event);
  380. }
  381. }
  382. /**
  383. * extract query parts and params array from data array
  384. *
  385. * @param array $data
  386. * @return array
  387. */
  388. protected function normalizeData(array $data): array {
  389. $fields = [
  390. 'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted',
  391. 'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size'];
  392. $extensionFields = ['metadata_etag', 'creation_time', 'upload_time'];
  393. $doNotCopyStorageMTime = false;
  394. if (array_key_exists('mtime', $data) && $data['mtime'] === null) {
  395. // this horrific magic tells it to not copy storage_mtime to mtime
  396. unset($data['mtime']);
  397. $doNotCopyStorageMTime = true;
  398. }
  399. $params = [];
  400. $extensionParams = [];
  401. foreach ($data as $name => $value) {
  402. if (in_array($name, $fields)) {
  403. if ($name === 'path') {
  404. $params['path_hash'] = md5($value);
  405. } elseif ($name === 'mimetype') {
  406. $params['mimepart'] = $this->mimetypeLoader->getId(substr($value, 0, strpos($value, '/')));
  407. $value = $this->mimetypeLoader->getId($value);
  408. } elseif ($name === 'storage_mtime') {
  409. if (!$doNotCopyStorageMTime && !isset($data['mtime'])) {
  410. $params['mtime'] = $value;
  411. }
  412. } elseif ($name === 'encrypted') {
  413. if (isset($data['encryptedVersion'])) {
  414. $value = $data['encryptedVersion'];
  415. } else {
  416. // Boolean to integer conversion
  417. $value = $value ? 1 : 0;
  418. }
  419. }
  420. $params[$name] = $value;
  421. }
  422. if (in_array($name, $extensionFields)) {
  423. $extensionParams[$name] = $value;
  424. }
  425. }
  426. return [$params, array_filter($extensionParams)];
  427. }
  428. /**
  429. * get the file id for a file
  430. *
  431. * A file id is a numeric id for a file or folder that's unique within an owncloud instance which stays the same for the lifetime of a file
  432. *
  433. * File ids are easiest way for apps to store references to a file since unlike paths they are not affected by renames or sharing
  434. *
  435. * @param string $file
  436. * @return int
  437. */
  438. public function getId($file) {
  439. // normalize file
  440. $file = $this->normalize($file);
  441. $query = $this->getQueryBuilder();
  442. $query->select('fileid')
  443. ->from('filecache')
  444. ->whereStorageId($this->getNumericStorageId())
  445. ->wherePath($file);
  446. $result = $query->execute();
  447. $id = $result->fetchOne();
  448. $result->closeCursor();
  449. return $id === false ? -1 : (int)$id;
  450. }
  451. /**
  452. * get the id of the parent folder of a file
  453. *
  454. * @param string $file
  455. * @return int
  456. */
  457. public function getParentId($file) {
  458. if ($file === '') {
  459. return -1;
  460. } else {
  461. $parent = $this->getParentPath($file);
  462. return (int)$this->getId($parent);
  463. }
  464. }
  465. private function getParentPath($path) {
  466. $parent = dirname($path);
  467. if ($parent === '.') {
  468. $parent = '';
  469. }
  470. return $parent;
  471. }
  472. /**
  473. * check if a file is available in the cache
  474. *
  475. * @param string $file
  476. * @return bool
  477. */
  478. public function inCache($file) {
  479. return $this->getId($file) != -1;
  480. }
  481. /**
  482. * remove a file or folder from the cache
  483. *
  484. * when removing a folder from the cache all files and folders inside the folder will be removed as well
  485. *
  486. * @param string $file
  487. */
  488. public function remove($file) {
  489. $entry = $this->get($file);
  490. if ($entry instanceof ICacheEntry) {
  491. $query = $this->getQueryBuilder();
  492. $query->delete('filecache')
  493. ->whereFileId($entry->getId());
  494. $query->execute();
  495. $query = $this->getQueryBuilder();
  496. $query->delete('filecache_extended')
  497. ->whereFileId($entry->getId());
  498. $query->execute();
  499. if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) {
  500. $this->removeChildren($entry);
  501. }
  502. $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $entry->getPath(), $entry->getId(), $this->getNumericStorageId()));
  503. }
  504. }
  505. /**
  506. * Remove all children of a folder
  507. *
  508. * @param ICacheEntry $entry the cache entry of the folder to remove the children of
  509. * @throws \OC\DatabaseException
  510. */
  511. private function removeChildren(ICacheEntry $entry) {
  512. $parentIds = [$entry->getId()];
  513. $queue = [$entry->getId()];
  514. $deletedIds = [];
  515. $deletedPaths = [];
  516. // we walk depth first through the file tree, removing all filecache_extended attributes while we walk
  517. // and collecting all folder ids to later use to delete the filecache entries
  518. while ($entryId = array_pop($queue)) {
  519. $children = $this->getFolderContentsById($entryId);
  520. $childIds = array_map(function (ICacheEntry $cacheEntry) {
  521. return $cacheEntry->getId();
  522. }, $children);
  523. $childPaths = array_map(function (ICacheEntry $cacheEntry) {
  524. return $cacheEntry->getPath();
  525. }, $children);
  526. foreach ($childIds as $childId) {
  527. $deletedIds[] = $childId;
  528. }
  529. foreach ($childPaths as $childPath) {
  530. $deletedPaths[] = $childPath;
  531. }
  532. $query = $this->getQueryBuilder();
  533. $query->delete('filecache_extended')
  534. ->where($query->expr()->in('fileid', $query->createParameter('childIds')));
  535. foreach (array_chunk($childIds, 1000) as $childIdChunk) {
  536. $query->setParameter('childIds', $childIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
  537. $query->execute();
  538. }
  539. /** @var ICacheEntry[] $childFolders */
  540. $childFolders = [];
  541. foreach ($children as $child) {
  542. if ($child->getMimeType() == FileInfo::MIMETYPE_FOLDER) {
  543. $childFolders[] = $child;
  544. }
  545. }
  546. foreach ($childFolders as $folder) {
  547. $parentIds[] = $folder->getId();
  548. $queue[] = $folder->getId();
  549. }
  550. }
  551. $query = $this->getQueryBuilder();
  552. $query->delete('filecache')
  553. ->whereParentInParameter('parentIds');
  554. // Sorting before chunking allows the db to find the entries close to each
  555. // other in the index
  556. sort($parentIds, SORT_NUMERIC);
  557. foreach (array_chunk($parentIds, 1000) as $parentIdChunk) {
  558. $query->setParameter('parentIds', $parentIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
  559. $query->execute();
  560. }
  561. foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) {
  562. $cacheEntryRemovedEvent = new CacheEntryRemovedEvent(
  563. $this->storage,
  564. $filePath,
  565. $fileId,
  566. $this->getNumericStorageId()
  567. );
  568. $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
  569. }
  570. }
  571. /**
  572. * Move a file or folder in the cache
  573. *
  574. * @param string $source
  575. * @param string $target
  576. */
  577. public function move($source, $target) {
  578. $this->moveFromCache($this, $source, $target);
  579. }
  580. /**
  581. * Get the storage id and path needed for a move
  582. *
  583. * @param string $path
  584. * @return array [$storageId, $internalPath]
  585. */
  586. protected function getMoveInfo($path) {
  587. return [$this->getNumericStorageId(), $path];
  588. }
  589. protected function hasEncryptionWrapper(): bool {
  590. return $this->storage->instanceOfStorage(Encryption::class);
  591. }
  592. /**
  593. * Move a file or folder in the cache
  594. *
  595. * @param ICache $sourceCache
  596. * @param string $sourcePath
  597. * @param string $targetPath
  598. * @throws \OC\DatabaseException
  599. * @throws \Exception if the given storages have an invalid id
  600. */
  601. public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
  602. if ($sourceCache instanceof Cache) {
  603. // normalize source and target
  604. $sourcePath = $this->normalize($sourcePath);
  605. $targetPath = $this->normalize($targetPath);
  606. $sourceData = $sourceCache->get($sourcePath);
  607. if (!$sourceData) {
  608. throw new \Exception('Invalid source storage path: ' . $sourcePath);
  609. }
  610. $sourceId = $sourceData['fileid'];
  611. $newParentId = $this->getParentId($targetPath);
  612. [$sourceStorageId, $sourcePath] = $sourceCache->getMoveInfo($sourcePath);
  613. [$targetStorageId, $targetPath] = $this->getMoveInfo($targetPath);
  614. if (is_null($sourceStorageId) || $sourceStorageId === false) {
  615. throw new \Exception('Invalid source storage id: ' . $sourceStorageId);
  616. }
  617. if (is_null($targetStorageId) || $targetStorageId === false) {
  618. throw new \Exception('Invalid target storage id: ' . $targetStorageId);
  619. }
  620. if ($sourceData['mimetype'] === 'httpd/unix-directory') {
  621. //update all child entries
  622. $sourceLength = mb_strlen($sourcePath);
  623. $query = $this->connection->getQueryBuilder();
  624. $fun = $query->func();
  625. $newPathFunction = $fun->concat(
  626. $query->createNamedParameter($targetPath),
  627. $fun->substring('path', $query->createNamedParameter($sourceLength + 1, IQueryBuilder::PARAM_INT))// +1 for the leading slash
  628. );
  629. $query->update('filecache')
  630. ->set('storage', $query->createNamedParameter($targetStorageId, IQueryBuilder::PARAM_INT))
  631. ->set('path_hash', $fun->md5($newPathFunction))
  632. ->set('path', $newPathFunction)
  633. ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT)))
  634. ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%')));
  635. // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
  636. if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
  637. $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT));
  638. }
  639. // Retry transaction in case of RetryableException like deadlocks.
  640. // Retry up to 4 times because we should receive up to 4 concurrent requests from the frontend
  641. $retryLimit = 4;
  642. for ($i = 1; $i <= $retryLimit; $i++) {
  643. try {
  644. $this->connection->beginTransaction();
  645. $query->executeStatement();
  646. break;
  647. } catch (\OC\DatabaseException $e) {
  648. $this->connection->rollBack();
  649. throw $e;
  650. } catch (RetryableException $e) {
  651. // Simply throw if we already retried 4 times.
  652. if ($i === $retryLimit) {
  653. throw $e;
  654. }
  655. $this->connection->rollBack();
  656. // Sleep a bit to give some time to the other transaction to finish.
  657. usleep(100 * 1000 * $i);
  658. }
  659. }
  660. } else {
  661. $this->connection->beginTransaction();
  662. }
  663. $query = $this->getQueryBuilder();
  664. $query->update('filecache')
  665. ->set('storage', $query->createNamedParameter($targetStorageId))
  666. ->set('path', $query->createNamedParameter($targetPath))
  667. ->set('path_hash', $query->createNamedParameter(md5($targetPath)))
  668. ->set('name', $query->createNamedParameter(basename($targetPath)))
  669. ->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT))
  670. ->whereFileId($sourceId);
  671. // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
  672. if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
  673. $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT));
  674. }
  675. $query->execute();
  676. $this->connection->commit();
  677. if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) {
  678. $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId()));
  679. $event = new CacheEntryInsertedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId());
  680. $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event);
  681. $this->eventDispatcher->dispatchTyped($event);
  682. } else {
  683. $event = new CacheEntryUpdatedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId());
  684. $this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event);
  685. $this->eventDispatcher->dispatchTyped($event);
  686. }
  687. } else {
  688. $this->moveFromCacheFallback($sourceCache, $sourcePath, $targetPath);
  689. }
  690. }
  691. /**
  692. * remove all entries for files that are stored on the storage from the cache
  693. */
  694. public function clear() {
  695. $query = $this->getQueryBuilder();
  696. $query->delete('filecache')
  697. ->whereStorageId($this->getNumericStorageId());
  698. $query->execute();
  699. $query = $this->connection->getQueryBuilder();
  700. $query->delete('storages')
  701. ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId)));
  702. $query->execute();
  703. }
  704. /**
  705. * Get the scan status of a file
  706. *
  707. * - Cache::NOT_FOUND: File is not in the cache
  708. * - Cache::PARTIAL: File is not stored in the cache but some incomplete data is known
  709. * - Cache::SHALLOW: The folder and it's direct children are in the cache but not all sub folders are fully scanned
  710. * - Cache::COMPLETE: The file or folder, with all it's children) are fully scanned
  711. *
  712. * @param string $file
  713. *
  714. * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE
  715. */
  716. public function getStatus($file) {
  717. // normalize file
  718. $file = $this->normalize($file);
  719. $query = $this->getQueryBuilder();
  720. $query->select('size')
  721. ->from('filecache')
  722. ->whereStorageId($this->getNumericStorageId())
  723. ->wherePath($file);
  724. $result = $query->execute();
  725. $size = $result->fetchOne();
  726. $result->closeCursor();
  727. if ($size !== false) {
  728. if ((int)$size === -1) {
  729. return self::SHALLOW;
  730. } else {
  731. return self::COMPLETE;
  732. }
  733. } else {
  734. if (isset($this->partial[$file])) {
  735. return self::PARTIAL;
  736. } else {
  737. return self::NOT_FOUND;
  738. }
  739. }
  740. }
  741. /**
  742. * search for files matching $pattern
  743. *
  744. * @param string $pattern the search pattern using SQL search syntax (e.g. '%searchstring%')
  745. * @return ICacheEntry[] an array of cache entries where the name matches the search pattern
  746. */
  747. public function search($pattern) {
  748. $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', $pattern);
  749. return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null));
  750. }
  751. /**
  752. * search for files by mimetype
  753. *
  754. * @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image')
  755. * where it will search for all mimetypes in the group ('image/*')
  756. * @return ICacheEntry[] an array of cache entries where the mimetype matches the search
  757. */
  758. public function searchByMime($mimetype) {
  759. if (!str_contains($mimetype, '/')) {
  760. $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%');
  761. } else {
  762. $operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype);
  763. }
  764. return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null));
  765. }
  766. public function searchQuery(ISearchQuery $searchQuery) {
  767. return current($this->querySearchHelper->searchInCaches($searchQuery, [$this]));
  768. }
  769. /**
  770. * Re-calculate the folder size and the size of all parent folders
  771. *
  772. * @param string|boolean $path
  773. * @param array $data (optional) meta data of the folder
  774. */
  775. public function correctFolderSize($path, $data = null, $isBackgroundScan = false) {
  776. $this->calculateFolderSize($path, $data);
  777. if ($path !== '') {
  778. $parent = dirname($path);
  779. if ($parent === '.' || $parent === '/') {
  780. $parent = '';
  781. }
  782. if ($isBackgroundScan) {
  783. $parentData = $this->get($parent);
  784. if ($parentData['size'] !== -1 && $this->getIncompleteChildrenCount($parentData['fileid']) === 0) {
  785. $this->correctFolderSize($parent, $parentData, $isBackgroundScan);
  786. }
  787. } else {
  788. $this->correctFolderSize($parent);
  789. }
  790. }
  791. }
  792. /**
  793. * get the incomplete count that shares parent $folder
  794. *
  795. * @param int $fileId the file id of the folder
  796. * @return int
  797. */
  798. public function getIncompleteChildrenCount($fileId) {
  799. if ($fileId > -1) {
  800. $query = $this->getQueryBuilder();
  801. $query->select($query->func()->count())
  802. ->from('filecache')
  803. ->whereParent($fileId)
  804. ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
  805. $result = $query->execute();
  806. $size = (int)$result->fetchOne();
  807. $result->closeCursor();
  808. return $size;
  809. }
  810. return -1;
  811. }
  812. /**
  813. * calculate the size of a folder and set it in the cache
  814. *
  815. * @param string $path
  816. * @param array|null|ICacheEntry $entry (optional) meta data of the folder
  817. * @return int|float
  818. */
  819. public function calculateFolderSize($path, $entry = null) {
  820. return $this->calculateFolderSizeInner($path, $entry);
  821. }
  822. /**
  823. * inner function because we can't add new params to the public function without breaking any child classes
  824. *
  825. * @param string $path
  826. * @param array|null|ICacheEntry $entry (optional) meta data of the folder
  827. * @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown
  828. * @return int|float
  829. */
  830. protected function calculateFolderSizeInner(string $path, $entry = null, bool $ignoreUnknown = false) {
  831. $totalSize = 0;
  832. if (is_null($entry) || !isset($entry['fileid'])) {
  833. $entry = $this->get($path);
  834. }
  835. if (isset($entry['mimetype']) && $entry['mimetype'] === FileInfo::MIMETYPE_FOLDER) {
  836. $id = $entry['fileid'];
  837. $query = $this->getQueryBuilder();
  838. $query->select('size', 'unencrypted_size')
  839. ->from('filecache')
  840. ->whereParent($id);
  841. if ($ignoreUnknown) {
  842. $query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0)));
  843. }
  844. $result = $query->execute();
  845. $rows = $result->fetchAll();
  846. $result->closeCursor();
  847. if ($rows) {
  848. $sizes = array_map(function (array $row) {
  849. return Util::numericToNumber($row['size']);
  850. }, $rows);
  851. $unencryptedOnlySizes = array_map(function (array $row) {
  852. return Util::numericToNumber($row['unencrypted_size']);
  853. }, $rows);
  854. $unencryptedSizes = array_map(function (array $row) {
  855. return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']);
  856. }, $rows);
  857. $sum = array_sum($sizes);
  858. $min = min($sizes);
  859. $unencryptedSum = array_sum($unencryptedSizes);
  860. $unencryptedMin = min($unencryptedSizes);
  861. $unencryptedMax = max($unencryptedOnlySizes);
  862. $sum = 0 + $sum;
  863. $min = 0 + $min;
  864. if ($min === -1) {
  865. $totalSize = $min;
  866. } else {
  867. $totalSize = $sum;
  868. }
  869. if ($unencryptedMin === -1 || $min === -1) {
  870. $unencryptedTotal = $unencryptedMin;
  871. } else {
  872. $unencryptedTotal = $unencryptedSum;
  873. }
  874. } else {
  875. $totalSize = 0;
  876. $unencryptedTotal = 0;
  877. $unencryptedMax = 0;
  878. }
  879. // only set unencrypted size for a folder if any child entries have it set, or the folder is empty
  880. $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || $entry['unencrypted_size'] > 0;
  881. if ($entry['size'] !== $totalSize || ($entry['unencrypted_size'] !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) {
  882. if ($shouldWriteUnEncryptedSize) {
  883. // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes
  884. if ($unencryptedMax === 0) {
  885. $unencryptedTotal = 0;
  886. }
  887. $this->update($id, [
  888. 'size' => $totalSize,
  889. 'unencrypted_size' => $unencryptedTotal,
  890. ]);
  891. } else {
  892. $this->update($id, [
  893. 'size' => $totalSize,
  894. ]);
  895. }
  896. }
  897. }
  898. return $totalSize;
  899. }
  900. /**
  901. * get all file ids on the files on the storage
  902. *
  903. * @return int[]
  904. */
  905. public function getAll() {
  906. $query = $this->getQueryBuilder();
  907. $query->select('fileid')
  908. ->from('filecache')
  909. ->whereStorageId($this->getNumericStorageId());
  910. $result = $query->execute();
  911. $files = $result->fetchAll(\PDO::FETCH_COLUMN);
  912. $result->closeCursor();
  913. return array_map(function ($id) {
  914. return (int)$id;
  915. }, $files);
  916. }
  917. /**
  918. * find a folder in the cache which has not been fully scanned
  919. *
  920. * If multiple incomplete folders are in the cache, the one with the highest id will be returned,
  921. * use the one with the highest id gives the best result with the background scanner, since that is most
  922. * likely the folder where we stopped scanning previously
  923. *
  924. * @return string|false the path of the folder or false when no folder matched
  925. */
  926. public function getIncomplete() {
  927. // we select the fileid here first instead of directly selecting the path since this helps mariadb/mysql
  928. // to use the correct index.
  929. // The overhead of this should be minimal since the cost of selecting the path by id should be much lower
  930. // than the cost of finding an item with size < 0
  931. $query = $this->getQueryBuilder();
  932. $query->select('fileid')
  933. ->from('filecache')
  934. ->whereStorageId($this->getNumericStorageId())
  935. ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
  936. ->orderBy('fileid', 'DESC')
  937. ->setMaxResults(1);
  938. $result = $query->execute();
  939. $id = $result->fetchOne();
  940. $result->closeCursor();
  941. if ($id === false) {
  942. return false;
  943. }
  944. $path = $this->getPathById($id);
  945. return $path ?? false;
  946. }
  947. /**
  948. * get the path of a file on this storage by it's file id
  949. *
  950. * @param int $id the file id of the file or folder to search
  951. * @return string|null the path of the file (relative to the storage) or null if a file with the given id does not exists within this cache
  952. */
  953. public function getPathById($id) {
  954. $query = $this->getQueryBuilder();
  955. $query->select('path')
  956. ->from('filecache')
  957. ->whereStorageId($this->getNumericStorageId())
  958. ->whereFileId($id);
  959. $result = $query->execute();
  960. $path = $result->fetchOne();
  961. $result->closeCursor();
  962. if ($path === false) {
  963. return null;
  964. }
  965. return (string)$path;
  966. }
  967. /**
  968. * get the storage id of the storage for a file and the internal path of the file
  969. * unlike getPathById this does not limit the search to files on this storage and
  970. * instead does a global search in the cache table
  971. *
  972. * @param int $id
  973. * @return array first element holding the storage id, second the path
  974. * @deprecated use getPathById() instead
  975. */
  976. public static function getById($id) {
  977. $query = \OC::$server->getDatabaseConnection()->getQueryBuilder();
  978. $query->select('path', 'storage')
  979. ->from('filecache')
  980. ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
  981. $result = $query->execute();
  982. $row = $result->fetch();
  983. $result->closeCursor();
  984. if ($row) {
  985. $numericId = $row['storage'];
  986. $path = $row['path'];
  987. } else {
  988. return null;
  989. }
  990. if ($id = Storage::getStorageId($numericId)) {
  991. return [$id, $path];
  992. } else {
  993. return null;
  994. }
  995. }
  996. /**
  997. * normalize the given path
  998. *
  999. * @param string $path
  1000. * @return string
  1001. */
  1002. public function normalize($path) {
  1003. return trim(\OC_Util::normalizeUnicode($path), '/');
  1004. }
  1005. /**
  1006. * Copy a file or folder in the cache
  1007. *
  1008. * @param ICache $sourceCache
  1009. * @param ICacheEntry $sourceEntry
  1010. * @param string $targetPath
  1011. * @return int fileId of copied entry
  1012. */
  1013. public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int {
  1014. if ($sourceEntry->getId() < 0) {
  1015. throw new \RuntimeException("Invalid source cache entry on copyFromCache");
  1016. }
  1017. $data = $this->cacheEntryToArray($sourceEntry);
  1018. // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
  1019. if ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
  1020. $data['encrypted'] = 0;
  1021. }
  1022. $fileId = $this->put($targetPath, $data);
  1023. if ($fileId <= 0) {
  1024. throw new \RuntimeException("Failed to copy to " . $targetPath . " from cache with source data " . json_encode($data) . " ");
  1025. }
  1026. if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  1027. $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId());
  1028. foreach ($folderContent as $subEntry) {
  1029. $subTargetPath = $targetPath . '/' . $subEntry->getName();
  1030. $this->copyFromCache($sourceCache, $subEntry, $subTargetPath);
  1031. }
  1032. }
  1033. return $fileId;
  1034. }
  1035. private function cacheEntryToArray(ICacheEntry $entry): array {
  1036. return [
  1037. 'size' => $entry->getSize(),
  1038. 'mtime' => $entry->getMTime(),
  1039. 'storage_mtime' => $entry->getStorageMTime(),
  1040. 'mimetype' => $entry->getMimeType(),
  1041. 'mimepart' => $entry->getMimePart(),
  1042. 'etag' => $entry->getEtag(),
  1043. 'permissions' => $entry->getPermissions(),
  1044. 'encrypted' => $entry->isEncrypted(),
  1045. 'creation_time' => $entry->getCreationTime(),
  1046. 'upload_time' => $entry->getUploadTime(),
  1047. 'metadata_etag' => $entry->getMetadataEtag(),
  1048. ];
  1049. }
  1050. public function getQueryFilterForStorage(): ISearchOperator {
  1051. return new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'storage', $this->getNumericStorageId());
  1052. }
  1053. public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry {
  1054. if ($rawEntry->getStorageId() === $this->getNumericStorageId()) {
  1055. return $rawEntry;
  1056. } else {
  1057. return null;
  1058. }
  1059. }
  1060. }