Cache.php 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162
  1. <?php
  2. /**
  3. * @copyright Copyright (c) 2016, ownCloud, Inc.
  4. *
  5. * @author Andreas Fischer <bantu@owncloud.com>
  6. * @author Ari Selseng <ari@selseng.net>
  7. * @author Artem Kochnev <MrJeos@gmail.com>
  8. * @author Björn Schießle <bjoern@schiessle.org>
  9. * @author Christoph Wurst <christoph@winzerhof-wurst.at>
  10. * @author Daniel Kesselberg <mail@danielkesselberg.de>
  11. * @author Florin Peter <github@florin-peter.de>
  12. * @author Frédéric Fortier <frederic.fortier@oronospolytechnique.com>
  13. * @author Jens-Christian Fischer <jens-christian.fischer@switch.ch>
  14. * @author Joas Schilling <coding@schilljs.com>
  15. * @author John Molakvoæ <skjnldsv@protonmail.com>
  16. * @author Jörn Friedrich Dreyer <jfd@butonic.de>
  17. * @author Lukas Reschke <lukas@statuscode.ch>
  18. * @author Michael Gapczynski <GapczynskiM@gmail.com>
  19. * @author Morris Jobke <hey@morrisjobke.de>
  20. * @author Robin Appelman <robin@icewind.nl>
  21. * @author Robin McCorkell <robin@mccorkell.me.uk>
  22. * @author Roeland Jago Douma <roeland@famdouma.nl>
  23. * @author Vincent Petry <vincent@nextcloud.com>
  24. *
  25. * @license AGPL-3.0
  26. *
  27. * This code is free software: you can redistribute it and/or modify
  28. * it under the terms of the GNU Affero General Public License, version 3,
  29. * as published by the Free Software Foundation.
  30. *
  31. * This program is distributed in the hope that it will be useful,
  32. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  33. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  34. * GNU Affero General Public License for more details.
  35. *
  36. * You should have received a copy of the GNU Affero General Public License, version 3,
  37. * along with this program. If not, see <http://www.gnu.org/licenses/>
  38. *
  39. */
  40. namespace OC\Files\Cache;
  41. use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
  42. use OC\Files\Search\SearchComparison;
  43. use OC\Files\Search\SearchQuery;
  44. use OC\Files\Storage\Wrapper\Encryption;
  45. use OCP\DB\QueryBuilder\IQueryBuilder;
  46. use OCP\EventDispatcher\IEventDispatcher;
  47. use OCP\Files\Cache\CacheEntryInsertedEvent;
  48. use OCP\Files\Cache\CacheEntryUpdatedEvent;
  49. use OCP\Files\Cache\CacheInsertEvent;
  50. use OCP\Files\Cache\CacheEntryRemovedEvent;
  51. use OCP\Files\Cache\CacheUpdateEvent;
  52. use OCP\Files\Cache\ICache;
  53. use OCP\Files\Cache\ICacheEntry;
  54. use OCP\Files\FileInfo;
  55. use OCP\Files\IMimeTypeLoader;
  56. use OCP\Files\Search\ISearchComparison;
  57. use OCP\Files\Search\ISearchOperator;
  58. use OCP\Files\Search\ISearchQuery;
  59. use OCP\Files\Storage\IStorage;
  60. use OCP\IDBConnection;
  61. use OCP\Util;
  62. use Psr\Log\LoggerInterface;
  63. /**
  64. * Metadata cache for a storage
  65. *
  66. * The cache stores the metadata for all files and folders in a storage and is kept up to date through the following mechanisms:
  67. *
  68. * - Scanner: scans the storage and updates the cache where needed
  69. * - Watcher: checks for changes made to the filesystem outside of the Nextcloud instance and rescans files and folder when a change is detected
  70. * - Updater: listens to changes made to the filesystem inside of the Nextcloud instance and updates the cache where needed
  71. * - ChangePropagator: updates the mtime and etags of parent folders whenever a change to the cache is made to the cache by the updater
  72. */
  73. class Cache implements ICache {
  74. use MoveFromCacheTrait {
  75. MoveFromCacheTrait::moveFromCache as moveFromCacheFallback;
  76. }
  77. /**
  78. * @var array partial data for the cache
  79. */
  80. protected $partial = [];
  81. /**
  82. * @var string
  83. */
  84. protected $storageId;
  85. private $storage;
  86. /**
  87. * @var Storage $storageCache
  88. */
  89. protected $storageCache;
  90. /** @var IMimeTypeLoader */
  91. protected $mimetypeLoader;
  92. /**
  93. * @var IDBConnection
  94. */
  95. protected $connection;
  96. /**
  97. * @var IEventDispatcher
  98. */
  99. protected $eventDispatcher;
  100. /** @var QuerySearchHelper */
  101. protected $querySearchHelper;
  102. /**
  103. * @param IStorage $storage
  104. */
  105. public function __construct(IStorage $storage) {
  106. $this->storageId = $storage->getId();
  107. $this->storage = $storage;
  108. if (strlen($this->storageId) > 64) {
  109. $this->storageId = md5($this->storageId);
  110. }
  111. $this->storageCache = new Storage($storage);
  112. $this->mimetypeLoader = \OC::$server->getMimeTypeLoader();
  113. $this->connection = \OC::$server->getDatabaseConnection();
  114. $this->eventDispatcher = \OC::$server->get(IEventDispatcher::class);
  115. $this->querySearchHelper = \OCP\Server::get(QuerySearchHelper::class);
  116. }
  117. protected function getQueryBuilder() {
  118. return new CacheQueryBuilder(
  119. $this->connection,
  120. \OC::$server->getSystemConfig(),
  121. \OC::$server->get(LoggerInterface::class)
  122. );
  123. }
  124. /**
  125. * Get the numeric storage id for this cache's storage
  126. *
  127. * @return int
  128. */
  129. public function getNumericStorageId() {
  130. return $this->storageCache->getNumericId();
  131. }
  132. /**
  133. * get the stored metadata of a file or folder
  134. *
  135. * @param string | int $file either the path of a file or folder or the file id for a file or folder
  136. * @return ICacheEntry|false the cache entry as array of false if the file is not found in the cache
  137. */
  138. public function get($file) {
  139. $query = $this->getQueryBuilder();
  140. $query->selectFileCache();
  141. if (is_string($file) || $file == '') {
  142. // normalize file
  143. $file = $this->normalize($file);
  144. $query->whereStorageId($this->getNumericStorageId())
  145. ->wherePath($file);
  146. } else { //file id
  147. $query->whereFileId($file);
  148. }
  149. $result = $query->execute();
  150. $data = $result->fetch();
  151. $result->closeCursor();
  152. //merge partial data
  153. if (!$data && is_string($file) && isset($this->partial[$file])) {
  154. return $this->partial[$file];
  155. } elseif (!$data) {
  156. return $data;
  157. } else {
  158. return self::cacheEntryFromData($data, $this->mimetypeLoader);
  159. }
  160. }
  161. /**
  162. * Create a CacheEntry from database row
  163. *
  164. * @param array $data
  165. * @param IMimeTypeLoader $mimetypeLoader
  166. * @return CacheEntry
  167. */
  168. public static function cacheEntryFromData($data, IMimeTypeLoader $mimetypeLoader) {
  169. //fix types
  170. $data['name'] = (string)$data['name'];
  171. $data['path'] = (string)$data['path'];
  172. $data['fileid'] = (int)$data['fileid'];
  173. $data['parent'] = (int)$data['parent'];
  174. $data['size'] = Util::numericToNumber($data['size']);
  175. $data['unencrypted_size'] = Util::numericToNumber($data['unencrypted_size'] ?? 0);
  176. $data['mtime'] = (int)$data['mtime'];
  177. $data['storage_mtime'] = (int)$data['storage_mtime'];
  178. $data['encryptedVersion'] = (int)$data['encrypted'];
  179. $data['encrypted'] = (bool)$data['encrypted'];
  180. $data['storage_id'] = $data['storage'];
  181. $data['storage'] = (int)$data['storage'];
  182. $data['mimetype'] = $mimetypeLoader->getMimetypeById($data['mimetype']);
  183. $data['mimepart'] = $mimetypeLoader->getMimetypeById($data['mimepart']);
  184. if ($data['storage_mtime'] == 0) {
  185. $data['storage_mtime'] = $data['mtime'];
  186. }
  187. $data['permissions'] = (int)$data['permissions'];
  188. if (isset($data['creation_time'])) {
  189. $data['creation_time'] = (int)$data['creation_time'];
  190. }
  191. if (isset($data['upload_time'])) {
  192. $data['upload_time'] = (int)$data['upload_time'];
  193. }
  194. return new CacheEntry($data);
  195. }
  196. /**
  197. * get the metadata of all files stored in $folder
  198. *
  199. * @param string $folder
  200. * @return ICacheEntry[]
  201. */
  202. public function getFolderContents($folder) {
  203. $fileId = $this->getId($folder);
  204. return $this->getFolderContentsById($fileId);
  205. }
  206. /**
  207. * get the metadata of all files stored in $folder
  208. *
  209. * @param int $fileId the file id of the folder
  210. * @return ICacheEntry[]
  211. */
  212. public function getFolderContentsById($fileId) {
  213. if ($fileId > -1) {
  214. $query = $this->getQueryBuilder();
  215. $query->selectFileCache()
  216. ->whereParent($fileId)
  217. ->orderBy('name', 'ASC');
  218. $result = $query->execute();
  219. $files = $result->fetchAll();
  220. $result->closeCursor();
  221. return array_map(function (array $data) {
  222. return self::cacheEntryFromData($data, $this->mimetypeLoader);
  223. }, $files);
  224. }
  225. return [];
  226. }
  227. /**
  228. * insert or update meta data for a file or folder
  229. *
  230. * @param string $file
  231. * @param array $data
  232. *
  233. * @return int file id
  234. * @throws \RuntimeException
  235. */
  236. public function put($file, array $data) {
  237. if (($id = $this->getId($file)) > -1) {
  238. $this->update($id, $data);
  239. return $id;
  240. } else {
  241. return $this->insert($file, $data);
  242. }
  243. }
  244. /**
  245. * insert meta data for a new file or folder
  246. *
  247. * @param string $file
  248. * @param array $data
  249. *
  250. * @return int file id
  251. * @throws \RuntimeException
  252. */
  253. public function insert($file, array $data) {
  254. // normalize file
  255. $file = $this->normalize($file);
  256. if (isset($this->partial[$file])) { //add any saved partial data
  257. $data = array_merge($this->partial[$file], $data);
  258. unset($this->partial[$file]);
  259. }
  260. $requiredFields = ['size', 'mtime', 'mimetype'];
  261. foreach ($requiredFields as $field) {
  262. if (!isset($data[$field])) { //data not complete save as partial and return
  263. $this->partial[$file] = $data;
  264. return -1;
  265. }
  266. }
  267. $data['path'] = $file;
  268. if (!isset($data['parent'])) {
  269. $data['parent'] = $this->getParentId($file);
  270. }
  271. $data['name'] = basename($file);
  272. [$values, $extensionValues] = $this->normalizeData($data);
  273. $storageId = $this->getNumericStorageId();
  274. $values['storage'] = $storageId;
  275. try {
  276. $builder = $this->connection->getQueryBuilder();
  277. $builder->insert('filecache');
  278. foreach ($values as $column => $value) {
  279. $builder->setValue($column, $builder->createNamedParameter($value));
  280. }
  281. if ($builder->execute()) {
  282. $fileId = $builder->getLastInsertId();
  283. if (count($extensionValues)) {
  284. $query = $this->getQueryBuilder();
  285. $query->insert('filecache_extended');
  286. $query->setValue('fileid', $query->createNamedParameter($fileId, IQueryBuilder::PARAM_INT));
  287. foreach ($extensionValues as $column => $value) {
  288. $query->setValue($column, $query->createNamedParameter($value));
  289. }
  290. $query->execute();
  291. }
  292. $event = new CacheEntryInsertedEvent($this->storage, $file, $fileId, $storageId);
  293. $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event);
  294. $this->eventDispatcher->dispatchTyped($event);
  295. return $fileId;
  296. }
  297. } catch (UniqueConstraintViolationException $e) {
  298. // entry exists already
  299. if ($this->connection->inTransaction()) {
  300. $this->connection->commit();
  301. $this->connection->beginTransaction();
  302. }
  303. }
  304. // The file was created in the mean time
  305. if (($id = $this->getId($file)) > -1) {
  306. $this->update($id, $data);
  307. return $id;
  308. } else {
  309. throw new \RuntimeException('File entry could not be inserted but could also not be selected with getId() in order to perform an update. Please try again.');
  310. }
  311. }
  312. /**
  313. * update the metadata of an existing file or folder in the cache
  314. *
  315. * @param int $id the fileid of the existing file or folder
  316. * @param array $data [$key => $value] the metadata to update, only the fields provided in the array will be updated, non-provided values will remain unchanged
  317. */
  318. public function update($id, array $data) {
  319. if (isset($data['path'])) {
  320. // normalize path
  321. $data['path'] = $this->normalize($data['path']);
  322. }
  323. if (isset($data['name'])) {
  324. // normalize path
  325. $data['name'] = $this->normalize($data['name']);
  326. }
  327. [$values, $extensionValues] = $this->normalizeData($data);
  328. if (count($values)) {
  329. $query = $this->getQueryBuilder();
  330. $query->update('filecache')
  331. ->whereFileId($id)
  332. ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) {
  333. return $query->expr()->orX(
  334. $query->expr()->neq($key, $query->createNamedParameter($value)),
  335. $query->expr()->isNull($key)
  336. );
  337. }, array_keys($values), array_values($values))));
  338. foreach ($values as $key => $value) {
  339. $query->set($key, $query->createNamedParameter($value));
  340. }
  341. $query->execute();
  342. }
  343. if (count($extensionValues)) {
  344. try {
  345. $query = $this->getQueryBuilder();
  346. $query->insert('filecache_extended');
  347. $query->setValue('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT));
  348. foreach ($extensionValues as $column => $value) {
  349. $query->setValue($column, $query->createNamedParameter($value));
  350. }
  351. $query->execute();
  352. } catch (UniqueConstraintViolationException $e) {
  353. $query = $this->getQueryBuilder();
  354. $query->update('filecache_extended')
  355. ->whereFileId($id)
  356. ->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) {
  357. return $query->expr()->orX(
  358. $query->expr()->neq($key, $query->createNamedParameter($value)),
  359. $query->expr()->isNull($key)
  360. );
  361. }, array_keys($extensionValues), array_values($extensionValues))));
  362. foreach ($extensionValues as $key => $value) {
  363. $query->set($key, $query->createNamedParameter($value));
  364. }
  365. $query->execute();
  366. }
  367. }
  368. $path = $this->getPathById($id);
  369. // path can still be null if the file doesn't exist
  370. if ($path !== null) {
  371. $event = new CacheEntryUpdatedEvent($this->storage, $path, $id, $this->getNumericStorageId());
  372. $this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event);
  373. $this->eventDispatcher->dispatchTyped($event);
  374. }
  375. }
  376. /**
  377. * extract query parts and params array from data array
  378. *
  379. * @param array $data
  380. * @return array
  381. */
  382. protected function normalizeData(array $data): array {
  383. $fields = [
  384. 'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted',
  385. 'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size'];
  386. $extensionFields = ['metadata_etag', 'creation_time', 'upload_time'];
  387. $doNotCopyStorageMTime = false;
  388. if (array_key_exists('mtime', $data) && $data['mtime'] === null) {
  389. // this horrific magic tells it to not copy storage_mtime to mtime
  390. unset($data['mtime']);
  391. $doNotCopyStorageMTime = true;
  392. }
  393. $params = [];
  394. $extensionParams = [];
  395. foreach ($data as $name => $value) {
  396. if (array_search($name, $fields) !== false) {
  397. if ($name === 'path') {
  398. $params['path_hash'] = md5($value);
  399. } elseif ($name === 'mimetype') {
  400. $params['mimepart'] = $this->mimetypeLoader->getId(substr($value, 0, strpos($value, '/')));
  401. $value = $this->mimetypeLoader->getId($value);
  402. } elseif ($name === 'storage_mtime') {
  403. if (!$doNotCopyStorageMTime && !isset($data['mtime'])) {
  404. $params['mtime'] = $value;
  405. }
  406. } elseif ($name === 'encrypted') {
  407. if (isset($data['encryptedVersion'])) {
  408. $value = $data['encryptedVersion'];
  409. } else {
  410. // Boolean to integer conversion
  411. $value = $value ? 1 : 0;
  412. }
  413. }
  414. $params[$name] = $value;
  415. }
  416. if (array_search($name, $extensionFields) !== false) {
  417. $extensionParams[$name] = $value;
  418. }
  419. }
  420. return [$params, array_filter($extensionParams)];
  421. }
  422. /**
  423. * get the file id for a file
  424. *
  425. * A file id is a numeric id for a file or folder that's unique within an owncloud instance which stays the same for the lifetime of a file
  426. *
  427. * File ids are easiest way for apps to store references to a file since unlike paths they are not affected by renames or sharing
  428. *
  429. * @param string $file
  430. * @return int
  431. */
  432. public function getId($file) {
  433. // normalize file
  434. $file = $this->normalize($file);
  435. $query = $this->getQueryBuilder();
  436. $query->select('fileid')
  437. ->from('filecache')
  438. ->whereStorageId($this->getNumericStorageId())
  439. ->wherePath($file);
  440. $result = $query->execute();
  441. $id = $result->fetchOne();
  442. $result->closeCursor();
  443. return $id === false ? -1 : (int)$id;
  444. }
  445. /**
  446. * get the id of the parent folder of a file
  447. *
  448. * @param string $file
  449. * @return int
  450. */
  451. public function getParentId($file) {
  452. if ($file === '') {
  453. return -1;
  454. } else {
  455. $parent = $this->getParentPath($file);
  456. return (int)$this->getId($parent);
  457. }
  458. }
  459. private function getParentPath($path) {
  460. $parent = dirname($path);
  461. if ($parent === '.') {
  462. $parent = '';
  463. }
  464. return $parent;
  465. }
  466. /**
  467. * check if a file is available in the cache
  468. *
  469. * @param string $file
  470. * @return bool
  471. */
  472. public function inCache($file) {
  473. return $this->getId($file) != -1;
  474. }
  475. /**
  476. * remove a file or folder from the cache
  477. *
  478. * when removing a folder from the cache all files and folders inside the folder will be removed as well
  479. *
  480. * @param string $file
  481. */
  482. public function remove($file) {
  483. $entry = $this->get($file);
  484. if ($entry instanceof ICacheEntry) {
  485. $query = $this->getQueryBuilder();
  486. $query->delete('filecache')
  487. ->whereFileId($entry->getId());
  488. $query->execute();
  489. $query = $this->getQueryBuilder();
  490. $query->delete('filecache_extended')
  491. ->whereFileId($entry->getId());
  492. $query->execute();
  493. if ($entry->getMimeType() == FileInfo::MIMETYPE_FOLDER) {
  494. $this->removeChildren($entry);
  495. }
  496. $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $entry->getPath(), $entry->getId(), $this->getNumericStorageId()));
  497. }
  498. }
  499. /**
  500. * Remove all children of a folder
  501. *
  502. * @param ICacheEntry $entry the cache entry of the folder to remove the children of
  503. * @throws \OC\DatabaseException
  504. */
  505. private function removeChildren(ICacheEntry $entry) {
  506. $parentIds = [$entry->getId()];
  507. $queue = [$entry->getId()];
  508. $deletedIds = [];
  509. $deletedPaths = [];
  510. // we walk depth first through the file tree, removing all filecache_extended attributes while we walk
  511. // and collecting all folder ids to later use to delete the filecache entries
  512. while ($entryId = array_pop($queue)) {
  513. $children = $this->getFolderContentsById($entryId);
  514. $childIds = array_map(function (ICacheEntry $cacheEntry) {
  515. return $cacheEntry->getId();
  516. }, $children);
  517. $childPaths = array_map(function (ICacheEntry $cacheEntry) {
  518. return $cacheEntry->getPath();
  519. }, $children);
  520. $deletedIds = array_merge($deletedIds, $childIds);
  521. $deletedPaths = array_merge($deletedPaths, $childPaths);
  522. $query = $this->getQueryBuilder();
  523. $query->delete('filecache_extended')
  524. ->where($query->expr()->in('fileid', $query->createParameter('childIds')));
  525. foreach (array_chunk($childIds, 1000) as $childIdChunk) {
  526. $query->setParameter('childIds', $childIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
  527. $query->execute();
  528. }
  529. /** @var ICacheEntry[] $childFolders */
  530. $childFolders = array_filter($children, function ($child) {
  531. return $child->getMimeType() == FileInfo::MIMETYPE_FOLDER;
  532. });
  533. foreach ($childFolders as $folder) {
  534. $parentIds[] = $folder->getId();
  535. $queue[] = $folder->getId();
  536. }
  537. }
  538. $query = $this->getQueryBuilder();
  539. $query->delete('filecache')
  540. ->whereParentInParameter('parentIds');
  541. foreach (array_chunk($parentIds, 1000) as $parentIdChunk) {
  542. $query->setParameter('parentIds', $parentIdChunk, IQueryBuilder::PARAM_INT_ARRAY);
  543. $query->execute();
  544. }
  545. foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) {
  546. $cacheEntryRemovedEvent = new CacheEntryRemovedEvent(
  547. $this->storage,
  548. $filePath,
  549. $fileId,
  550. $this->getNumericStorageId()
  551. );
  552. $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
  553. }
  554. }
  555. /**
  556. * Move a file or folder in the cache
  557. *
  558. * @param string $source
  559. * @param string $target
  560. */
  561. public function move($source, $target) {
  562. $this->moveFromCache($this, $source, $target);
  563. }
  564. /**
  565. * Get the storage id and path needed for a move
  566. *
  567. * @param string $path
  568. * @return array [$storageId, $internalPath]
  569. */
  570. protected function getMoveInfo($path) {
  571. return [$this->getNumericStorageId(), $path];
  572. }
  573. protected function hasEncryptionWrapper(): bool {
  574. return $this->storage->instanceOfStorage(Encryption::class);
  575. }
  576. /**
  577. * Move a file or folder in the cache
  578. *
  579. * @param ICache $sourceCache
  580. * @param string $sourcePath
  581. * @param string $targetPath
  582. * @throws \OC\DatabaseException
  583. * @throws \Exception if the given storages have an invalid id
  584. */
  585. public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) {
  586. if ($sourceCache instanceof Cache) {
  587. // normalize source and target
  588. $sourcePath = $this->normalize($sourcePath);
  589. $targetPath = $this->normalize($targetPath);
  590. $sourceData = $sourceCache->get($sourcePath);
  591. if (!$sourceData) {
  592. throw new \Exception('Invalid source storage path: ' . $sourcePath);
  593. }
  594. $sourceId = $sourceData['fileid'];
  595. $newParentId = $this->getParentId($targetPath);
  596. [$sourceStorageId, $sourcePath] = $sourceCache->getMoveInfo($sourcePath);
  597. [$targetStorageId, $targetPath] = $this->getMoveInfo($targetPath);
  598. if (is_null($sourceStorageId) || $sourceStorageId === false) {
  599. throw new \Exception('Invalid source storage id: ' . $sourceStorageId);
  600. }
  601. if (is_null($targetStorageId) || $targetStorageId === false) {
  602. throw new \Exception('Invalid target storage id: ' . $targetStorageId);
  603. }
  604. $this->connection->beginTransaction();
  605. if ($sourceData['mimetype'] === 'httpd/unix-directory') {
  606. //update all child entries
  607. $sourceLength = mb_strlen($sourcePath);
  608. $query = $this->connection->getQueryBuilder();
  609. $fun = $query->func();
  610. $newPathFunction = $fun->concat(
  611. $query->createNamedParameter($targetPath),
  612. $fun->substring('path', $query->createNamedParameter($sourceLength + 1, IQueryBuilder::PARAM_INT))// +1 for the leading slash
  613. );
  614. $query->update('filecache')
  615. ->set('storage', $query->createNamedParameter($targetStorageId, IQueryBuilder::PARAM_INT))
  616. ->set('path_hash', $fun->md5($newPathFunction))
  617. ->set('path', $newPathFunction)
  618. ->where($query->expr()->eq('storage', $query->createNamedParameter($sourceStorageId, IQueryBuilder::PARAM_INT)))
  619. ->andWhere($query->expr()->like('path', $query->createNamedParameter($this->connection->escapeLikeParameter($sourcePath) . '/%')));
  620. // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
  621. if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
  622. $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT));
  623. }
  624. try {
  625. $query->execute();
  626. } catch (\OC\DatabaseException $e) {
  627. $this->connection->rollBack();
  628. throw $e;
  629. }
  630. }
  631. $query = $this->getQueryBuilder();
  632. $query->update('filecache')
  633. ->set('storage', $query->createNamedParameter($targetStorageId))
  634. ->set('path', $query->createNamedParameter($targetPath))
  635. ->set('path_hash', $query->createNamedParameter(md5($targetPath)))
  636. ->set('name', $query->createNamedParameter(basename($targetPath)))
  637. ->set('parent', $query->createNamedParameter($newParentId, IQueryBuilder::PARAM_INT))
  638. ->whereFileId($sourceId);
  639. // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
  640. if ($sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
  641. $query->set('encrypted', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT));
  642. }
  643. $query->execute();
  644. $this->connection->commit();
  645. if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) {
  646. $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId()));
  647. $event = new CacheEntryInsertedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId());
  648. $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event);
  649. $this->eventDispatcher->dispatchTyped($event);
  650. } else {
  651. $event = new CacheEntryUpdatedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId());
  652. $this->eventDispatcher->dispatch(CacheUpdateEvent::class, $event);
  653. $this->eventDispatcher->dispatchTyped($event);
  654. }
  655. } else {
  656. $this->moveFromCacheFallback($sourceCache, $sourcePath, $targetPath);
  657. }
  658. }
  659. /**
  660. * remove all entries for files that are stored on the storage from the cache
  661. */
  662. public function clear() {
  663. $query = $this->getQueryBuilder();
  664. $query->delete('filecache')
  665. ->whereStorageId($this->getNumericStorageId());
  666. $query->execute();
  667. $query = $this->connection->getQueryBuilder();
  668. $query->delete('storages')
  669. ->where($query->expr()->eq('id', $query->createNamedParameter($this->storageId)));
  670. $query->execute();
  671. }
  672. /**
  673. * Get the scan status of a file
  674. *
  675. * - Cache::NOT_FOUND: File is not in the cache
  676. * - Cache::PARTIAL: File is not stored in the cache but some incomplete data is known
  677. * - Cache::SHALLOW: The folder and it's direct children are in the cache but not all sub folders are fully scanned
  678. * - Cache::COMPLETE: The file or folder, with all it's children) are fully scanned
  679. *
  680. * @param string $file
  681. *
  682. * @return int Cache::NOT_FOUND, Cache::PARTIAL, Cache::SHALLOW or Cache::COMPLETE
  683. */
  684. public function getStatus($file) {
  685. // normalize file
  686. $file = $this->normalize($file);
  687. $query = $this->getQueryBuilder();
  688. $query->select('size')
  689. ->from('filecache')
  690. ->whereStorageId($this->getNumericStorageId())
  691. ->wherePath($file);
  692. $result = $query->execute();
  693. $size = $result->fetchOne();
  694. $result->closeCursor();
  695. if ($size !== false) {
  696. if ((int)$size === -1) {
  697. return self::SHALLOW;
  698. } else {
  699. return self::COMPLETE;
  700. }
  701. } else {
  702. if (isset($this->partial[$file])) {
  703. return self::PARTIAL;
  704. } else {
  705. return self::NOT_FOUND;
  706. }
  707. }
  708. }
  709. /**
  710. * search for files matching $pattern
  711. *
  712. * @param string $pattern the search pattern using SQL search syntax (e.g. '%searchstring%')
  713. * @return ICacheEntry[] an array of cache entries where the name matches the search pattern
  714. */
  715. public function search($pattern) {
  716. $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'name', $pattern);
  717. return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null));
  718. }
  719. /**
  720. * search for files by mimetype
  721. *
  722. * @param string $mimetype either a full mimetype to search ('text/plain') or only the first part of a mimetype ('image')
  723. * where it will search for all mimetypes in the group ('image/*')
  724. * @return ICacheEntry[] an array of cache entries where the mimetype matches the search
  725. */
  726. public function searchByMime($mimetype) {
  727. if (!str_contains($mimetype, '/')) {
  728. $operator = new SearchComparison(ISearchComparison::COMPARE_LIKE, 'mimetype', $mimetype . '/%');
  729. } else {
  730. $operator = new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'mimetype', $mimetype);
  731. }
  732. return $this->searchQuery(new SearchQuery($operator, 0, 0, [], null));
  733. }
  734. public function searchQuery(ISearchQuery $searchQuery) {
  735. return current($this->querySearchHelper->searchInCaches($searchQuery, [$this]));
  736. }
  737. /**
  738. * Re-calculate the folder size and the size of all parent folders
  739. *
  740. * @param string|boolean $path
  741. * @param array $data (optional) meta data of the folder
  742. */
  743. public function correctFolderSize($path, $data = null, $isBackgroundScan = false) {
  744. $this->calculateFolderSize($path, $data);
  745. if ($path !== '') {
  746. $parent = dirname($path);
  747. if ($parent === '.' || $parent === '/') {
  748. $parent = '';
  749. }
  750. if ($isBackgroundScan) {
  751. $parentData = $this->get($parent);
  752. if ($parentData['size'] !== -1 && $this->getIncompleteChildrenCount($parentData['fileid']) === 0) {
  753. $this->correctFolderSize($parent, $parentData, $isBackgroundScan);
  754. }
  755. } else {
  756. $this->correctFolderSize($parent);
  757. }
  758. }
  759. }
  760. /**
  761. * get the incomplete count that shares parent $folder
  762. *
  763. * @param int $fileId the file id of the folder
  764. * @return int
  765. */
  766. public function getIncompleteChildrenCount($fileId) {
  767. if ($fileId > -1) {
  768. $query = $this->getQueryBuilder();
  769. $query->select($query->func()->count())
  770. ->from('filecache')
  771. ->whereParent($fileId)
  772. ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
  773. $result = $query->execute();
  774. $size = (int)$result->fetchOne();
  775. $result->closeCursor();
  776. return $size;
  777. }
  778. return -1;
  779. }
  780. /**
  781. * calculate the size of a folder and set it in the cache
  782. *
  783. * @param string $path
  784. * @param array|null|ICacheEntry $entry (optional) meta data of the folder
  785. * @return int|float
  786. */
  787. public function calculateFolderSize($path, $entry = null) {
  788. return $this->calculateFolderSizeInner($path, $entry);
  789. }
  790. /**
  791. * inner function because we can't add new params to the public function without breaking any child classes
  792. *
  793. * @param string $path
  794. * @param array|null|ICacheEntry $entry (optional) meta data of the folder
  795. * @param bool $ignoreUnknown don't mark the folder size as unknown if any of it's children are unknown
  796. * @return int|float
  797. */
  798. protected function calculateFolderSizeInner(string $path, $entry = null, bool $ignoreUnknown = false) {
  799. $totalSize = 0;
  800. if (is_null($entry) || !isset($entry['fileid'])) {
  801. $entry = $this->get($path);
  802. }
  803. if (isset($entry['mimetype']) && $entry['mimetype'] === FileInfo::MIMETYPE_FOLDER) {
  804. $id = $entry['fileid'];
  805. $query = $this->getQueryBuilder();
  806. $query->select('size', 'unencrypted_size')
  807. ->from('filecache')
  808. ->whereParent($id);
  809. if ($ignoreUnknown) {
  810. $query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0)));
  811. }
  812. $result = $query->execute();
  813. $rows = $result->fetchAll();
  814. $result->closeCursor();
  815. if ($rows) {
  816. $sizes = array_map(function (array $row) {
  817. return Util::numericToNumber($row['size']);
  818. }, $rows);
  819. $unencryptedOnlySizes = array_map(function (array $row) {
  820. return Util::numericToNumber($row['unencrypted_size']);
  821. }, $rows);
  822. $unencryptedSizes = array_map(function (array $row) {
  823. return Util::numericToNumber(($row['unencrypted_size'] > 0) ? $row['unencrypted_size'] : $row['size']);
  824. }, $rows);
  825. $sum = array_sum($sizes);
  826. $min = min($sizes);
  827. $unencryptedSum = array_sum($unencryptedSizes);
  828. $unencryptedMin = min($unencryptedSizes);
  829. $unencryptedMax = max($unencryptedOnlySizes);
  830. $sum = 0 + $sum;
  831. $min = 0 + $min;
  832. if ($min === -1) {
  833. $totalSize = $min;
  834. } else {
  835. $totalSize = $sum;
  836. }
  837. if ($unencryptedMin === -1 || $min === -1) {
  838. $unencryptedTotal = $unencryptedMin;
  839. } else {
  840. $unencryptedTotal = $unencryptedSum;
  841. }
  842. } else {
  843. $totalSize = 0;
  844. $unencryptedTotal = 0;
  845. $unencryptedMax = 0;
  846. }
  847. // only set unencrypted size for a folder if any child entries have it set, or the folder is empty
  848. $shouldWriteUnEncryptedSize = $unencryptedMax > 0 || $totalSize === 0 || $entry['unencrypted_size'] > 0;
  849. if ($entry['size'] !== $totalSize || ($entry['unencrypted_size'] !== $unencryptedTotal && $shouldWriteUnEncryptedSize)) {
  850. if ($shouldWriteUnEncryptedSize) {
  851. // if all children have an unencrypted size of 0, just set the folder unencrypted size to 0 instead of summing the sizes
  852. if ($unencryptedMax === 0) {
  853. $unencryptedTotal = 0;
  854. }
  855. $this->update($id, [
  856. 'size' => $totalSize,
  857. 'unencrypted_size' => $unencryptedTotal,
  858. ]);
  859. } else {
  860. $this->update($id, [
  861. 'size' => $totalSize,
  862. ]);
  863. }
  864. }
  865. }
  866. return $totalSize;
  867. }
  868. /**
  869. * get all file ids on the files on the storage
  870. *
  871. * @return int[]
  872. */
  873. public function getAll() {
  874. $query = $this->getQueryBuilder();
  875. $query->select('fileid')
  876. ->from('filecache')
  877. ->whereStorageId($this->getNumericStorageId());
  878. $result = $query->execute();
  879. $files = $result->fetchAll(\PDO::FETCH_COLUMN);
  880. $result->closeCursor();
  881. return array_map(function ($id) {
  882. return (int)$id;
  883. }, $files);
  884. }
  885. /**
  886. * find a folder in the cache which has not been fully scanned
  887. *
  888. * If multiple incomplete folders are in the cache, the one with the highest id will be returned,
  889. * use the one with the highest id gives the best result with the background scanner, since that is most
  890. * likely the folder where we stopped scanning previously
  891. *
  892. * @return string|false the path of the folder or false when no folder matched
  893. */
  894. public function getIncomplete() {
  895. $query = $this->getQueryBuilder();
  896. $query->select('path')
  897. ->from('filecache')
  898. ->whereStorageId($this->getNumericStorageId())
  899. ->andWhere($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
  900. ->orderBy('fileid', 'DESC')
  901. ->setMaxResults(1);
  902. $result = $query->execute();
  903. $path = $result->fetchOne();
  904. $result->closeCursor();
  905. if ($path === false) {
  906. return false;
  907. }
  908. // Make sure Oracle does not continue with null for empty strings
  909. return (string)$path;
  910. }
  911. /**
  912. * get the path of a file on this storage by it's file id
  913. *
  914. * @param int $id the file id of the file or folder to search
  915. * @return string|null the path of the file (relative to the storage) or null if a file with the given id does not exists within this cache
  916. */
  917. public function getPathById($id) {
  918. $query = $this->getQueryBuilder();
  919. $query->select('path')
  920. ->from('filecache')
  921. ->whereStorageId($this->getNumericStorageId())
  922. ->whereFileId($id);
  923. $result = $query->execute();
  924. $path = $result->fetchOne();
  925. $result->closeCursor();
  926. if ($path === false) {
  927. return null;
  928. }
  929. return (string)$path;
  930. }
  931. /**
  932. * get the storage id of the storage for a file and the internal path of the file
  933. * unlike getPathById this does not limit the search to files on this storage and
  934. * instead does a global search in the cache table
  935. *
  936. * @param int $id
  937. * @return array first element holding the storage id, second the path
  938. * @deprecated use getPathById() instead
  939. */
  940. public static function getById($id) {
  941. $query = \OC::$server->getDatabaseConnection()->getQueryBuilder();
  942. $query->select('path', 'storage')
  943. ->from('filecache')
  944. ->where($query->expr()->eq('fileid', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT)));
  945. $result = $query->execute();
  946. $row = $result->fetch();
  947. $result->closeCursor();
  948. if ($row) {
  949. $numericId = $row['storage'];
  950. $path = $row['path'];
  951. } else {
  952. return null;
  953. }
  954. if ($id = Storage::getStorageId($numericId)) {
  955. return [$id, $path];
  956. } else {
  957. return null;
  958. }
  959. }
  960. /**
  961. * normalize the given path
  962. *
  963. * @param string $path
  964. * @return string
  965. */
  966. public function normalize($path) {
  967. return trim(\OC_Util::normalizeUnicode($path), '/');
  968. }
  969. /**
  970. * Copy a file or folder in the cache
  971. *
  972. * @param ICache $sourceCache
  973. * @param ICacheEntry $sourceEntry
  974. * @param string $targetPath
  975. * @return int fileId of copied entry
  976. */
  977. public function copyFromCache(ICache $sourceCache, ICacheEntry $sourceEntry, string $targetPath): int {
  978. if ($sourceEntry->getId() < 0) {
  979. throw new \RuntimeException("Invalid source cache entry on copyFromCache");
  980. }
  981. $data = $this->cacheEntryToArray($sourceEntry);
  982. // when moving from an encrypted storage to a non-encrypted storage remove the `encrypted` mark
  983. if ($sourceCache instanceof Cache && $sourceCache->hasEncryptionWrapper() && !$this->hasEncryptionWrapper()) {
  984. $data['encrypted'] = 0;
  985. }
  986. $fileId = $this->put($targetPath, $data);
  987. if ($fileId <= 0) {
  988. throw new \RuntimeException("Failed to copy to " . $targetPath . " from cache with source data " . json_encode($data) . " ");
  989. }
  990. if ($sourceEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  991. $folderContent = $sourceCache->getFolderContentsById($sourceEntry->getId());
  992. foreach ($folderContent as $subEntry) {
  993. $subTargetPath = $targetPath . '/' . $subEntry->getName();
  994. $this->copyFromCache($sourceCache, $subEntry, $subTargetPath);
  995. }
  996. }
  997. return $fileId;
  998. }
  999. private function cacheEntryToArray(ICacheEntry $entry): array {
  1000. return [
  1001. 'size' => $entry->getSize(),
  1002. 'mtime' => $entry->getMTime(),
  1003. 'storage_mtime' => $entry->getStorageMTime(),
  1004. 'mimetype' => $entry->getMimeType(),
  1005. 'mimepart' => $entry->getMimePart(),
  1006. 'etag' => $entry->getEtag(),
  1007. 'permissions' => $entry->getPermissions(),
  1008. 'encrypted' => $entry->isEncrypted(),
  1009. 'creation_time' => $entry->getCreationTime(),
  1010. 'upload_time' => $entry->getUploadTime(),
  1011. 'metadata_etag' => $entry->getMetadataEtag(),
  1012. ];
  1013. }
  1014. public function getQueryFilterForStorage(): ISearchOperator {
  1015. return new SearchComparison(ISearchComparison::COMPARE_EQUAL, 'storage', $this->getNumericStorageId());
  1016. }
  1017. public function getCacheEntryFromSearchResult(ICacheEntry $rawEntry): ?ICacheEntry {
  1018. if ($rawEntry->getStorageId() === $this->getNumericStorageId()) {
  1019. return $rawEntry;
  1020. } else {
  1021. return null;
  1022. }
  1023. }
  1024. }