ObjectStoreStorage.php 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795
  1. <?php
  2. /**
  3. * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
  4. * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
  5. * SPDX-License-Identifier: AGPL-3.0-only
  6. */
  7. namespace OC\Files\ObjectStore;
  8. use Aws\S3\Exception\S3Exception;
  9. use Aws\S3\Exception\S3MultipartUploadException;
  10. use Icewind\Streams\CallbackWrapper;
  11. use Icewind\Streams\CountWrapper;
  12. use Icewind\Streams\IteratorDirectory;
  13. use OC\Files\Cache\Cache;
  14. use OC\Files\Cache\CacheEntry;
  15. use OC\Files\Storage\PolyFill\CopyDirectory;
  16. use OCP\Files\Cache\ICache;
  17. use OCP\Files\Cache\ICacheEntry;
  18. use OCP\Files\Cache\IScanner;
  19. use OCP\Files\FileInfo;
  20. use OCP\Files\GenericFileException;
  21. use OCP\Files\NotFoundException;
  22. use OCP\Files\ObjectStore\IObjectStore;
  23. use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
  24. use OCP\Files\Storage\IChunkedFileWrite;
  25. use OCP\Files\Storage\IStorage;
  26. use Psr\Log\LoggerInterface;
  27. class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite {
  28. use CopyDirectory;
  29. protected IObjectStore $objectStore;
  30. protected string $id;
  31. private string $objectPrefix = 'urn:oid:';
  32. private LoggerInterface $logger;
  33. private bool $handleCopiesAsOwned;
  34. protected bool $validateWrites = true;
  35. private bool $preserveCacheItemsOnDelete = false;
  36. /**
  37. * @param array $parameters
  38. * @throws \Exception
  39. */
  40. public function __construct(array $parameters) {
  41. if (isset($parameters['objectstore']) && $parameters['objectstore'] instanceof IObjectStore) {
  42. $this->objectStore = $parameters['objectstore'];
  43. } else {
  44. throw new \Exception('missing IObjectStore instance');
  45. }
  46. if (isset($parameters['storageid'])) {
  47. $this->id = 'object::store:' . $parameters['storageid'];
  48. } else {
  49. $this->id = 'object::store:' . $this->objectStore->getStorageId();
  50. }
  51. if (isset($parameters['objectPrefix'])) {
  52. $this->objectPrefix = $parameters['objectPrefix'];
  53. }
  54. if (isset($parameters['validateWrites'])) {
  55. $this->validateWrites = (bool)$parameters['validateWrites'];
  56. }
  57. $this->handleCopiesAsOwned = (bool)($parameters['handleCopiesAsOwned'] ?? false);
  58. $this->logger = \OCP\Server::get(LoggerInterface::class);
  59. }
  60. public function mkdir(string $path, bool $force = false): bool {
  61. $path = $this->normalizePath($path);
  62. if (!$force && $this->file_exists($path)) {
  63. $this->logger->warning("Tried to create an object store folder that already exists: $path");
  64. return false;
  65. }
  66. $mTime = time();
  67. $data = [
  68. 'mimetype' => 'httpd/unix-directory',
  69. 'size' => 0,
  70. 'mtime' => $mTime,
  71. 'storage_mtime' => $mTime,
  72. 'permissions' => \OCP\Constants::PERMISSION_ALL,
  73. ];
  74. if ($path === '') {
  75. //create root on the fly
  76. $data['etag'] = $this->getETag('');
  77. $this->getCache()->put('', $data);
  78. return true;
  79. } else {
  80. // if parent does not exist, create it
  81. $parent = $this->normalizePath(dirname($path));
  82. $parentType = $this->filetype($parent);
  83. if ($parentType === false) {
  84. if (!$this->mkdir($parent)) {
  85. // something went wrong
  86. $this->logger->warning("Parent folder ($parent) doesn't exist and couldn't be created");
  87. return false;
  88. }
  89. } elseif ($parentType === 'file') {
  90. // parent is a file
  91. $this->logger->warning("Parent ($parent) is a file");
  92. return false;
  93. }
  94. // finally create the new dir
  95. $mTime = time(); // update mtime
  96. $data['mtime'] = $mTime;
  97. $data['storage_mtime'] = $mTime;
  98. $data['etag'] = $this->getETag($path);
  99. $this->getCache()->put($path, $data);
  100. return true;
  101. }
  102. }
  103. private function normalizePath(string $path): string {
  104. $path = trim($path, '/');
  105. //FIXME why do we sometimes get a path like 'files//username'?
  106. $path = str_replace('//', '/', $path);
  107. // dirname('/folder') returns '.' but internally (in the cache) we store the root as ''
  108. if (!$path || $path === '.') {
  109. $path = '';
  110. }
  111. return $path;
  112. }
  113. /**
  114. * Object Stores use a NoopScanner because metadata is directly stored in
  115. * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
  116. */
  117. public function getScanner(string $path = '', ?IStorage $storage = null): IScanner {
  118. if (!$storage) {
  119. $storage = $this;
  120. }
  121. if (!isset($this->scanner)) {
  122. $this->scanner = new ObjectStoreScanner($storage);
  123. }
  124. /** @var \OC\Files\ObjectStore\ObjectStoreScanner */
  125. return $this->scanner;
  126. }
  127. public function getId(): string {
  128. return $this->id;
  129. }
  130. public function rmdir(string $path): bool {
  131. $path = $this->normalizePath($path);
  132. $entry = $this->getCache()->get($path);
  133. if (!$entry || $entry->getMimeType() !== ICacheEntry::DIRECTORY_MIMETYPE) {
  134. return false;
  135. }
  136. return $this->rmObjects($entry);
  137. }
  138. private function rmObjects(ICacheEntry $entry): bool {
  139. $children = $this->getCache()->getFolderContentsById($entry->getId());
  140. foreach ($children as $child) {
  141. if ($child->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  142. if (!$this->rmObjects($child)) {
  143. return false;
  144. }
  145. } else {
  146. if (!$this->rmObject($child)) {
  147. return false;
  148. }
  149. }
  150. }
  151. if (!$this->preserveCacheItemsOnDelete) {
  152. $this->getCache()->remove($entry->getPath());
  153. }
  154. return true;
  155. }
  156. public function unlink(string $path): bool {
  157. $path = $this->normalizePath($path);
  158. $entry = $this->getCache()->get($path);
  159. if ($entry instanceof ICacheEntry) {
  160. if ($entry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  161. return $this->rmObjects($entry);
  162. } else {
  163. return $this->rmObject($entry);
  164. }
  165. }
  166. return false;
  167. }
  168. public function rmObject(ICacheEntry $entry): bool {
  169. try {
  170. $this->objectStore->deleteObject($this->getURN($entry->getId()));
  171. } catch (\Exception $ex) {
  172. if ($ex->getCode() !== 404) {
  173. $this->logger->error(
  174. 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(),
  175. [
  176. 'app' => 'objectstore',
  177. 'exception' => $ex,
  178. ]
  179. );
  180. return false;
  181. }
  182. //removing from cache is ok as it does not exist in the objectstore anyway
  183. }
  184. if (!$this->preserveCacheItemsOnDelete) {
  185. $this->getCache()->remove($entry->getPath());
  186. }
  187. return true;
  188. }
  189. public function stat(string $path): array|false {
  190. $path = $this->normalizePath($path);
  191. $cacheEntry = $this->getCache()->get($path);
  192. if ($cacheEntry instanceof CacheEntry) {
  193. return $cacheEntry->getData();
  194. } else {
  195. if ($path === '') {
  196. $this->mkdir('', true);
  197. $cacheEntry = $this->getCache()->get($path);
  198. if ($cacheEntry instanceof CacheEntry) {
  199. return $cacheEntry->getData();
  200. }
  201. }
  202. return false;
  203. }
  204. }
  205. public function getPermissions(string $path): int {
  206. $stat = $this->stat($path);
  207. if (is_array($stat) && isset($stat['permissions'])) {
  208. return $stat['permissions'];
  209. }
  210. return parent::getPermissions($path);
  211. }
  212. /**
  213. * Override this method if you need a different unique resource identifier for your object storage implementation.
  214. * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
  215. * You may need a mapping table to store your URN if it cannot be generated from the fileid.
  216. *
  217. * @return string the unified resource name used to identify the object
  218. */
  219. public function getURN(int $fileId): string {
  220. return $this->objectPrefix . $fileId;
  221. }
  222. public function opendir(string $path) {
  223. $path = $this->normalizePath($path);
  224. try {
  225. $files = [];
  226. $folderContents = $this->getCache()->getFolderContents($path);
  227. foreach ($folderContents as $file) {
  228. $files[] = $file['name'];
  229. }
  230. return IteratorDirectory::wrap($files);
  231. } catch (\Exception $e) {
  232. $this->logger->error($e->getMessage(), ['exception' => $e]);
  233. return false;
  234. }
  235. }
  236. public function filetype(string $path): string|false {
  237. $path = $this->normalizePath($path);
  238. $stat = $this->stat($path);
  239. if ($stat) {
  240. if ($stat['mimetype'] === 'httpd/unix-directory') {
  241. return 'dir';
  242. }
  243. return 'file';
  244. } else {
  245. return false;
  246. }
  247. }
  248. public function fopen(string $path, string $mode) {
  249. $path = $this->normalizePath($path);
  250. if (strrpos($path, '.') !== false) {
  251. $ext = substr($path, strrpos($path, '.'));
  252. } else {
  253. $ext = '';
  254. }
  255. switch ($mode) {
  256. case 'r':
  257. case 'rb':
  258. $stat = $this->stat($path);
  259. if (is_array($stat)) {
  260. $filesize = $stat['size'] ?? 0;
  261. // Reading 0 sized files is a waste of time
  262. if ($filesize === 0) {
  263. return fopen('php://memory', $mode);
  264. }
  265. try {
  266. $handle = $this->objectStore->readObject($this->getURN($stat['fileid']));
  267. if ($handle === false) {
  268. return false; // keep backward compatibility
  269. }
  270. $streamStat = fstat($handle);
  271. $actualSize = $streamStat['size'] ?? -1;
  272. if ($actualSize > -1 && $actualSize !== $filesize) {
  273. $this->getCache()->update((int)$stat['fileid'], ['size' => $actualSize]);
  274. }
  275. return $handle;
  276. } catch (NotFoundException $e) {
  277. $this->logger->error(
  278. 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
  279. [
  280. 'app' => 'objectstore',
  281. 'exception' => $e,
  282. ]
  283. );
  284. throw $e;
  285. } catch (\Exception $e) {
  286. $this->logger->error(
  287. 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
  288. [
  289. 'app' => 'objectstore',
  290. 'exception' => $e,
  291. ]
  292. );
  293. return false;
  294. }
  295. } else {
  296. return false;
  297. }
  298. // no break
  299. case 'w':
  300. case 'wb':
  301. case 'w+':
  302. case 'wb+':
  303. $dirName = dirname($path);
  304. $parentExists = $this->is_dir($dirName);
  305. if (!$parentExists) {
  306. return false;
  307. }
  308. $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
  309. $handle = fopen($tmpFile, $mode);
  310. return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
  311. $this->writeBack($tmpFile, $path);
  312. unlink($tmpFile);
  313. });
  314. case 'a':
  315. case 'ab':
  316. case 'r+':
  317. case 'a+':
  318. case 'x':
  319. case 'x+':
  320. case 'c':
  321. case 'c+':
  322. $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
  323. if ($this->file_exists($path)) {
  324. $source = $this->fopen($path, 'r');
  325. file_put_contents($tmpFile, $source);
  326. }
  327. $handle = fopen($tmpFile, $mode);
  328. return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
  329. $this->writeBack($tmpFile, $path);
  330. unlink($tmpFile);
  331. });
  332. }
  333. return false;
  334. }
  335. public function file_exists(string $path): bool {
  336. $path = $this->normalizePath($path);
  337. return (bool)$this->stat($path);
  338. }
  339. public function rename(string $source, string $target): bool {
  340. $source = $this->normalizePath($source);
  341. $target = $this->normalizePath($target);
  342. $this->remove($target);
  343. $this->getCache()->move($source, $target);
  344. $this->touch(dirname($target));
  345. return true;
  346. }
  347. public function getMimeType(string $path): string|false {
  348. $path = $this->normalizePath($path);
  349. return parent::getMimeType($path);
  350. }
  351. public function touch(string $path, ?int $mtime = null): bool {
  352. if (is_null($mtime)) {
  353. $mtime = time();
  354. }
  355. $path = $this->normalizePath($path);
  356. $dirName = dirname($path);
  357. $parentExists = $this->is_dir($dirName);
  358. if (!$parentExists) {
  359. return false;
  360. }
  361. $stat = $this->stat($path);
  362. if (is_array($stat)) {
  363. // update existing mtime in db
  364. $stat['mtime'] = $mtime;
  365. $this->getCache()->update($stat['fileid'], $stat);
  366. } else {
  367. try {
  368. //create a empty file, need to have at least on char to make it
  369. // work with all object storage implementations
  370. $this->file_put_contents($path, ' ');
  371. $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
  372. $stat = [
  373. 'etag' => $this->getETag($path),
  374. 'mimetype' => $mimeType,
  375. 'size' => 0,
  376. 'mtime' => $mtime,
  377. 'storage_mtime' => $mtime,
  378. 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
  379. ];
  380. $this->getCache()->put($path, $stat);
  381. } catch (\Exception $ex) {
  382. $this->logger->error(
  383. 'Could not create object for ' . $path,
  384. [
  385. 'app' => 'objectstore',
  386. 'exception' => $ex,
  387. ]
  388. );
  389. throw $ex;
  390. }
  391. }
  392. return true;
  393. }
  394. public function writeBack(string $tmpFile, string $path) {
  395. $size = filesize($tmpFile);
  396. $this->writeStream($path, fopen($tmpFile, 'r'), $size);
  397. }
  398. public function hasUpdated(string $path, int $time): bool {
  399. return false;
  400. }
  401. public function needsPartFile(): bool {
  402. return false;
  403. }
  404. public function file_put_contents(string $path, mixed $data): int {
  405. $fh = fopen('php://temp', 'w+');
  406. fwrite($fh, $data);
  407. rewind($fh);
  408. return $this->writeStream($path, $fh, strlen($data));
  409. }
  410. public function writeStream(string $path, $stream, ?int $size = null): int {
  411. $stat = $this->stat($path);
  412. if (empty($stat)) {
  413. // create new file
  414. $stat = [
  415. 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
  416. ];
  417. }
  418. // update stat with new data
  419. $mTime = time();
  420. $stat['size'] = (int)$size;
  421. $stat['mtime'] = $mTime;
  422. $stat['storage_mtime'] = $mTime;
  423. $mimetypeDetector = \OC::$server->getMimeTypeDetector();
  424. $mimetype = $mimetypeDetector->detectPath($path);
  425. $stat['mimetype'] = $mimetype;
  426. $stat['etag'] = $this->getETag($path);
  427. $stat['checksum'] = '';
  428. $exists = $this->getCache()->inCache($path);
  429. $uploadPath = $exists ? $path : $path . '.part';
  430. if ($exists) {
  431. $fileId = $stat['fileid'];
  432. } else {
  433. $parent = $this->normalizePath(dirname($path));
  434. if (!$this->is_dir($parent)) {
  435. throw new \InvalidArgumentException("trying to upload a file ($path) inside a non-directory ($parent)");
  436. }
  437. $fileId = $this->getCache()->put($uploadPath, $stat);
  438. }
  439. $urn = $this->getURN($fileId);
  440. try {
  441. //upload to object storage
  442. if ($size === null) {
  443. $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) {
  444. $this->getCache()->update($fileId, [
  445. 'size' => $writtenSize,
  446. ]);
  447. $size = $writtenSize;
  448. });
  449. $this->objectStore->writeObject($urn, $countStream, $mimetype);
  450. if (is_resource($countStream)) {
  451. fclose($countStream);
  452. }
  453. $stat['size'] = $size;
  454. } else {
  455. $this->objectStore->writeObject($urn, $stream, $mimetype);
  456. if (is_resource($stream)) {
  457. fclose($stream);
  458. }
  459. }
  460. } catch (\Exception $ex) {
  461. if (!$exists) {
  462. /*
  463. * Only remove the entry if we are dealing with a new file.
  464. * Else people lose access to existing files
  465. */
  466. $this->getCache()->remove($uploadPath);
  467. $this->logger->error(
  468. 'Could not create object ' . $urn . ' for ' . $path,
  469. [
  470. 'app' => 'objectstore',
  471. 'exception' => $ex,
  472. ]
  473. );
  474. } else {
  475. $this->logger->error(
  476. 'Could not update object ' . $urn . ' for ' . $path,
  477. [
  478. 'app' => 'objectstore',
  479. 'exception' => $ex,
  480. ]
  481. );
  482. }
  483. throw $ex; // make this bubble up
  484. }
  485. if ($exists) {
  486. // Always update the unencrypted size, for encryption the Encryption wrapper will update this afterwards anyways
  487. $stat['unencrypted_size'] = $stat['size'];
  488. $this->getCache()->update($fileId, $stat);
  489. } else {
  490. if (!$this->validateWrites || $this->objectStore->objectExists($urn)) {
  491. $this->getCache()->move($uploadPath, $path);
  492. } else {
  493. $this->getCache()->remove($uploadPath);
  494. throw new \Exception("Object not found after writing (urn: $urn, path: $path)", 404);
  495. }
  496. }
  497. return $size;
  498. }
  499. public function getObjectStore(): IObjectStore {
  500. return $this->objectStore;
  501. }
  502. public function copyFromStorage(
  503. IStorage $sourceStorage,
  504. string $sourceInternalPath,
  505. string $targetInternalPath,
  506. bool $preserveMtime = false,
  507. ): bool {
  508. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  509. /** @var ObjectStoreStorage $sourceStorage */
  510. if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) {
  511. /** @var CacheEntry $sourceEntry */
  512. $sourceEntry = $sourceStorage->getCache()->get($sourceInternalPath);
  513. $sourceEntryData = $sourceEntry->getData();
  514. // $sourceEntry['permissions'] here is the permissions from the jailed storage for the current
  515. // user. Instead we use $sourceEntryData['scan_permissions'] that are the permissions from the
  516. // unjailed storage.
  517. if (is_array($sourceEntryData) && array_key_exists('scan_permissions', $sourceEntryData)) {
  518. $sourceEntry['permissions'] = $sourceEntryData['scan_permissions'];
  519. }
  520. $this->copyInner($sourceStorage->getCache(), $sourceEntry, $targetInternalPath);
  521. return true;
  522. }
  523. }
  524. return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
  525. }
  526. public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool {
  527. $sourceCache = $sourceStorage->getCache();
  528. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class) && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) {
  529. $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath);
  530. // Do not import any data when source and target bucket are identical.
  531. return true;
  532. }
  533. if (!$sourceCacheEntry) {
  534. $sourceCacheEntry = $sourceCache->get($sourceInternalPath);
  535. }
  536. $this->copyObjects($sourceStorage, $sourceCache, $sourceCacheEntry);
  537. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  538. /** @var ObjectStoreStorage $sourceStorage */
  539. $sourceStorage->setPreserveCacheOnDelete(true);
  540. }
  541. if ($sourceCacheEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  542. $sourceStorage->rmdir($sourceInternalPath);
  543. } else {
  544. $sourceStorage->unlink($sourceInternalPath);
  545. }
  546. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  547. /** @var ObjectStoreStorage $sourceStorage */
  548. $sourceStorage->setPreserveCacheOnDelete(false);
  549. }
  550. $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath);
  551. return true;
  552. }
  553. /**
  554. * Copy the object(s) of a file or folder into this storage, without touching the cache
  555. */
  556. private function copyObjects(IStorage $sourceStorage, ICache $sourceCache, ICacheEntry $sourceCacheEntry) {
  557. $copiedFiles = [];
  558. try {
  559. foreach ($this->getAllChildObjects($sourceCache, $sourceCacheEntry) as $file) {
  560. $sourceStream = $sourceStorage->fopen($file->getPath(), 'r');
  561. if (!$sourceStream) {
  562. throw new \Exception("Failed to open source file {$file->getPath()} ({$file->getId()})");
  563. }
  564. $this->objectStore->writeObject($this->getURN($file->getId()), $sourceStream, $file->getMimeType());
  565. if (is_resource($sourceStream)) {
  566. fclose($sourceStream);
  567. }
  568. $copiedFiles[] = $file->getId();
  569. }
  570. } catch (\Exception $e) {
  571. foreach ($copiedFiles as $fileId) {
  572. try {
  573. $this->objectStore->deleteObject($this->getURN($fileId));
  574. } catch (\Exception $e) {
  575. // ignore
  576. }
  577. }
  578. throw $e;
  579. }
  580. }
  581. /**
  582. * @return \Iterator<ICacheEntry>
  583. */
  584. private function getAllChildObjects(ICache $cache, ICacheEntry $entry): \Iterator {
  585. if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
  586. foreach ($cache->getFolderContentsById($entry->getId()) as $child) {
  587. yield from $this->getAllChildObjects($cache, $child);
  588. }
  589. } else {
  590. yield $entry;
  591. }
  592. }
  593. public function copy(string $source, string $target): bool {
  594. $source = $this->normalizePath($source);
  595. $target = $this->normalizePath($target);
  596. $cache = $this->getCache();
  597. $sourceEntry = $cache->get($source);
  598. if (!$sourceEntry) {
  599. throw new NotFoundException('Source object not found');
  600. }
  601. $this->copyInner($cache, $sourceEntry, $target);
  602. return true;
  603. }
  604. private function copyInner(ICache $sourceCache, ICacheEntry $sourceEntry, string $to) {
  605. $cache = $this->getCache();
  606. if ($sourceEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
  607. if ($cache->inCache($to)) {
  608. $cache->remove($to);
  609. }
  610. $this->mkdir($to);
  611. foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) {
  612. $this->copyInner($sourceCache, $child, $to . '/' . $child->getName());
  613. }
  614. } else {
  615. $this->copyFile($sourceEntry, $to);
  616. }
  617. }
  618. private function copyFile(ICacheEntry $sourceEntry, string $to) {
  619. $cache = $this->getCache();
  620. $sourceUrn = $this->getURN($sourceEntry->getId());
  621. if (!$cache instanceof Cache) {
  622. throw new \Exception('Invalid source cache for object store copy');
  623. }
  624. $targetId = $cache->copyFromCache($cache, $sourceEntry, $to);
  625. $targetUrn = $this->getURN($targetId);
  626. try {
  627. $this->objectStore->copyObject($sourceUrn, $targetUrn);
  628. if ($this->handleCopiesAsOwned) {
  629. // Copied the file thus we gain all permissions as we are the owner now ! warning while this aligns with local storage it should not be used and instead fix local storage !
  630. $cache->update($targetId, ['permissions' => \OCP\Constants::PERMISSION_ALL]);
  631. }
  632. } catch (\Exception $e) {
  633. $cache->remove($to);
  634. throw $e;
  635. }
  636. }
  637. public function startChunkedWrite(string $targetPath): string {
  638. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  639. throw new GenericFileException('Object store does not support multipart upload');
  640. }
  641. $cacheEntry = $this->getCache()->get($targetPath);
  642. $urn = $this->getURN($cacheEntry->getId());
  643. return $this->objectStore->initiateMultipartUpload($urn);
  644. }
  645. /**
  646. * @throws GenericFileException
  647. */
  648. public function putChunkedWritePart(
  649. string $targetPath,
  650. string $writeToken,
  651. string $chunkId,
  652. $data,
  653. $size = null,
  654. ): ?array {
  655. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  656. throw new GenericFileException('Object store does not support multipart upload');
  657. }
  658. $cacheEntry = $this->getCache()->get($targetPath);
  659. $urn = $this->getURN($cacheEntry->getId());
  660. $result = $this->objectStore->uploadMultipartPart($urn, $writeToken, (int)$chunkId, $data, $size);
  661. $parts[$chunkId] = [
  662. 'PartNumber' => $chunkId,
  663. 'ETag' => trim($result->get('ETag'), '"'),
  664. ];
  665. return $parts[$chunkId];
  666. }
  667. public function completeChunkedWrite(string $targetPath, string $writeToken): int {
  668. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  669. throw new GenericFileException('Object store does not support multipart upload');
  670. }
  671. $cacheEntry = $this->getCache()->get($targetPath);
  672. $urn = $this->getURN($cacheEntry->getId());
  673. $parts = $this->objectStore->getMultipartUploads($urn, $writeToken);
  674. $sortedParts = array_values($parts);
  675. sort($sortedParts);
  676. try {
  677. $size = $this->objectStore->completeMultipartUpload($urn, $writeToken, $sortedParts);
  678. $stat = $this->stat($targetPath);
  679. $mtime = time();
  680. if (is_array($stat)) {
  681. $stat['size'] = $size;
  682. $stat['mtime'] = $mtime;
  683. $stat['mimetype'] = $this->getMimeType($targetPath);
  684. $this->getCache()->update($stat['fileid'], $stat);
  685. }
  686. } catch (S3MultipartUploadException|S3Exception $e) {
  687. $this->objectStore->abortMultipartUpload($urn, $writeToken);
  688. $this->logger->error(
  689. 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken,
  690. [
  691. 'app' => 'objectstore',
  692. 'exception' => $e,
  693. ]
  694. );
  695. throw new GenericFileException('Could not write chunked file');
  696. }
  697. return $size;
  698. }
  699. public function cancelChunkedWrite(string $targetPath, string $writeToken): void {
  700. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  701. throw new GenericFileException('Object store does not support multipart upload');
  702. }
  703. $cacheEntry = $this->getCache()->get($targetPath);
  704. $urn = $this->getURN($cacheEntry->getId());
  705. $this->objectStore->abortMultipartUpload($urn, $writeToken);
  706. }
  707. public function setPreserveCacheOnDelete(bool $preserve) {
  708. $this->preserveCacheItemsOnDelete = $preserve;
  709. }
  710. }