ObjectStoreStorage.php 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802
  1. <?php
  2. /**
  3. * SPDX-FileCopyrightText: 2016-2024 Nextcloud GmbH and Nextcloud contributors
  4. * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
  5. * SPDX-License-Identifier: AGPL-3.0-only
  6. */
  7. namespace OC\Files\ObjectStore;
  8. use Aws\S3\Exception\S3Exception;
  9. use Aws\S3\Exception\S3MultipartUploadException;
  10. use Icewind\Streams\CallbackWrapper;
  11. use Icewind\Streams\CountWrapper;
  12. use Icewind\Streams\IteratorDirectory;
  13. use OC\Files\Cache\Cache;
  14. use OC\Files\Cache\CacheEntry;
  15. use OC\Files\Storage\PolyFill\CopyDirectory;
  16. use OCP\Files\Cache\ICache;
  17. use OCP\Files\Cache\ICacheEntry;
  18. use OCP\Files\Cache\IScanner;
  19. use OCP\Files\FileInfo;
  20. use OCP\Files\GenericFileException;
  21. use OCP\Files\NotFoundException;
  22. use OCP\Files\ObjectStore\IObjectStore;
  23. use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
  24. use OCP\Files\Storage\IChunkedFileWrite;
  25. use OCP\Files\Storage\IStorage;
  26. use Psr\Log\LoggerInterface;
  27. class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite {
  28. use CopyDirectory;
  29. protected IObjectStore $objectStore;
  30. protected string $id;
  31. private string $objectPrefix = 'urn:oid:';
  32. private LoggerInterface $logger;
  33. private bool $handleCopiesAsOwned;
  34. protected bool $validateWrites = true;
  35. private bool $preserveCacheItemsOnDelete = false;
  36. /**
  37. * @param array $parameters
  38. * @throws \Exception
  39. */
  40. public function __construct(array $parameters) {
  41. if (isset($parameters['objectstore']) && $parameters['objectstore'] instanceof IObjectStore) {
  42. $this->objectStore = $parameters['objectstore'];
  43. } else {
  44. throw new \Exception('missing IObjectStore instance');
  45. }
  46. if (isset($parameters['storageid'])) {
  47. $this->id = 'object::store:' . $parameters['storageid'];
  48. } else {
  49. $this->id = 'object::store:' . $this->objectStore->getStorageId();
  50. }
  51. if (isset($parameters['objectPrefix'])) {
  52. $this->objectPrefix = $parameters['objectPrefix'];
  53. }
  54. if (isset($parameters['validateWrites'])) {
  55. $this->validateWrites = (bool)$parameters['validateWrites'];
  56. }
  57. $this->handleCopiesAsOwned = (bool)($parameters['handleCopiesAsOwned'] ?? false);
  58. $this->logger = \OCP\Server::get(LoggerInterface::class);
  59. }
  60. public function mkdir(string $path, bool $force = false): bool {
  61. $path = $this->normalizePath($path);
  62. if (!$force && $this->file_exists($path)) {
  63. $this->logger->warning("Tried to create an object store folder that already exists: $path");
  64. return false;
  65. }
  66. $mTime = time();
  67. $data = [
  68. 'mimetype' => 'httpd/unix-directory',
  69. 'size' => 0,
  70. 'mtime' => $mTime,
  71. 'storage_mtime' => $mTime,
  72. 'permissions' => \OCP\Constants::PERMISSION_ALL,
  73. ];
  74. if ($path === '') {
  75. //create root on the fly
  76. $data['etag'] = $this->getETag('');
  77. $this->getCache()->put('', $data);
  78. return true;
  79. } else {
  80. // if parent does not exist, create it
  81. $parent = $this->normalizePath(dirname($path));
  82. $parentType = $this->filetype($parent);
  83. if ($parentType === false) {
  84. if (!$this->mkdir($parent)) {
  85. // something went wrong
  86. $this->logger->warning("Parent folder ($parent) doesn't exist and couldn't be created");
  87. return false;
  88. }
  89. } elseif ($parentType === 'file') {
  90. // parent is a file
  91. $this->logger->warning("Parent ($parent) is a file");
  92. return false;
  93. }
  94. // finally create the new dir
  95. $mTime = time(); // update mtime
  96. $data['mtime'] = $mTime;
  97. $data['storage_mtime'] = $mTime;
  98. $data['etag'] = $this->getETag($path);
  99. $this->getCache()->put($path, $data);
  100. return true;
  101. }
  102. }
  103. private function normalizePath(string $path): string {
  104. $path = trim($path, '/');
  105. //FIXME why do we sometimes get a path like 'files//username'?
  106. $path = str_replace('//', '/', $path);
  107. // dirname('/folder') returns '.' but internally (in the cache) we store the root as ''
  108. if (!$path || $path === '.') {
  109. $path = '';
  110. }
  111. return $path;
  112. }
  113. /**
  114. * Object Stores use a NoopScanner because metadata is directly stored in
  115. * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
  116. */
  117. public function getScanner(string $path = '', ?IStorage $storage = null): IScanner {
  118. if (!$storage) {
  119. $storage = $this;
  120. }
  121. if (!isset($this->scanner)) {
  122. $this->scanner = new ObjectStoreScanner($storage);
  123. }
  124. /** @var \OC\Files\ObjectStore\ObjectStoreScanner */
  125. return $this->scanner;
  126. }
  127. public function getId(): string {
  128. return $this->id;
  129. }
  130. public function rmdir(string $path): bool {
  131. $path = $this->normalizePath($path);
  132. $entry = $this->getCache()->get($path);
  133. if (!$entry || $entry->getMimeType() !== ICacheEntry::DIRECTORY_MIMETYPE) {
  134. return false;
  135. }
  136. return $this->rmObjects($entry);
  137. }
  138. private function rmObjects(ICacheEntry $entry): bool {
  139. $children = $this->getCache()->getFolderContentsById($entry->getId());
  140. foreach ($children as $child) {
  141. if ($child->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  142. if (!$this->rmObjects($child)) {
  143. return false;
  144. }
  145. } else {
  146. if (!$this->rmObject($child)) {
  147. return false;
  148. }
  149. }
  150. }
  151. if (!$this->preserveCacheItemsOnDelete) {
  152. $this->getCache()->remove($entry->getPath());
  153. }
  154. return true;
  155. }
  156. public function unlink(string $path): bool {
  157. $path = $this->normalizePath($path);
  158. $entry = $this->getCache()->get($path);
  159. if ($entry instanceof ICacheEntry) {
  160. if ($entry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  161. return $this->rmObjects($entry);
  162. } else {
  163. return $this->rmObject($entry);
  164. }
  165. }
  166. return false;
  167. }
  168. public function rmObject(ICacheEntry $entry): bool {
  169. try {
  170. $this->objectStore->deleteObject($this->getURN($entry->getId()));
  171. } catch (\Exception $ex) {
  172. if ($ex->getCode() !== 404) {
  173. $this->logger->error(
  174. 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(),
  175. [
  176. 'app' => 'objectstore',
  177. 'exception' => $ex,
  178. ]
  179. );
  180. return false;
  181. }
  182. //removing from cache is ok as it does not exist in the objectstore anyway
  183. }
  184. if (!$this->preserveCacheItemsOnDelete) {
  185. $this->getCache()->remove($entry->getPath());
  186. }
  187. return true;
  188. }
  189. public function stat(string $path): array|false {
  190. $path = $this->normalizePath($path);
  191. $cacheEntry = $this->getCache()->get($path);
  192. if ($cacheEntry instanceof CacheEntry) {
  193. return $cacheEntry->getData();
  194. } else {
  195. if ($path === '') {
  196. $this->mkdir('', true);
  197. $cacheEntry = $this->getCache()->get($path);
  198. if ($cacheEntry instanceof CacheEntry) {
  199. return $cacheEntry->getData();
  200. }
  201. }
  202. return false;
  203. }
  204. }
  205. public function getPermissions(string $path): int {
  206. $stat = $this->stat($path);
  207. if (is_array($stat) && isset($stat['permissions'])) {
  208. return $stat['permissions'];
  209. }
  210. return parent::getPermissions($path);
  211. }
  212. /**
  213. * Override this method if you need a different unique resource identifier for your object storage implementation.
  214. * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
  215. * You may need a mapping table to store your URN if it cannot be generated from the fileid.
  216. *
  217. * @return string the unified resource name used to identify the object
  218. */
  219. public function getURN(int $fileId): string {
  220. return $this->objectPrefix . $fileId;
  221. }
  222. public function opendir(string $path) {
  223. $path = $this->normalizePath($path);
  224. try {
  225. $files = [];
  226. $folderContents = $this->getCache()->getFolderContents($path);
  227. foreach ($folderContents as $file) {
  228. $files[] = $file['name'];
  229. }
  230. return IteratorDirectory::wrap($files);
  231. } catch (\Exception $e) {
  232. $this->logger->error($e->getMessage(), ['exception' => $e]);
  233. return false;
  234. }
  235. }
  236. public function filetype(string $path): string|false {
  237. $path = $this->normalizePath($path);
  238. $stat = $this->stat($path);
  239. if ($stat) {
  240. if ($stat['mimetype'] === 'httpd/unix-directory') {
  241. return 'dir';
  242. }
  243. return 'file';
  244. } else {
  245. return false;
  246. }
  247. }
  248. public function fopen(string $path, string $mode) {
  249. $path = $this->normalizePath($path);
  250. if (strrpos($path, '.') !== false) {
  251. $ext = substr($path, strrpos($path, '.'));
  252. } else {
  253. $ext = '';
  254. }
  255. switch ($mode) {
  256. case 'r':
  257. case 'rb':
  258. $stat = $this->stat($path);
  259. if (is_array($stat)) {
  260. $filesize = $stat['size'] ?? 0;
  261. // Reading 0 sized files is a waste of time
  262. if ($filesize === 0) {
  263. return fopen('php://memory', $mode);
  264. }
  265. try {
  266. $handle = $this->objectStore->readObject($this->getURN($stat['fileid']));
  267. if ($handle === false) {
  268. return false; // keep backward compatibility
  269. }
  270. $streamStat = fstat($handle);
  271. $actualSize = $streamStat['size'] ?? -1;
  272. if ($actualSize > -1 && $actualSize !== $filesize) {
  273. $this->getCache()->update((int)$stat['fileid'], ['size' => $actualSize]);
  274. }
  275. return $handle;
  276. } catch (NotFoundException $e) {
  277. $this->logger->error(
  278. 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
  279. [
  280. 'app' => 'objectstore',
  281. 'exception' => $e,
  282. ]
  283. );
  284. throw $e;
  285. } catch (\Exception $e) {
  286. $this->logger->error(
  287. 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
  288. [
  289. 'app' => 'objectstore',
  290. 'exception' => $e,
  291. ]
  292. );
  293. return false;
  294. }
  295. } else {
  296. return false;
  297. }
  298. // no break
  299. case 'w':
  300. case 'wb':
  301. case 'w+':
  302. case 'wb+':
  303. $dirName = dirname($path);
  304. $parentExists = $this->is_dir($dirName);
  305. if (!$parentExists) {
  306. return false;
  307. }
  308. $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
  309. $handle = fopen($tmpFile, $mode);
  310. return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
  311. $this->writeBack($tmpFile, $path);
  312. unlink($tmpFile);
  313. });
  314. case 'a':
  315. case 'ab':
  316. case 'r+':
  317. case 'a+':
  318. case 'x':
  319. case 'x+':
  320. case 'c':
  321. case 'c+':
  322. $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
  323. if ($this->file_exists($path)) {
  324. $source = $this->fopen($path, 'r');
  325. file_put_contents($tmpFile, $source);
  326. }
  327. $handle = fopen($tmpFile, $mode);
  328. return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
  329. $this->writeBack($tmpFile, $path);
  330. unlink($tmpFile);
  331. });
  332. }
  333. return false;
  334. }
  335. public function file_exists(string $path): bool {
  336. $path = $this->normalizePath($path);
  337. return (bool)$this->stat($path);
  338. }
  339. public function rename(string $source, string $target): bool {
  340. $source = $this->normalizePath($source);
  341. $target = $this->normalizePath($target);
  342. $this->remove($target);
  343. $this->getCache()->move($source, $target);
  344. $this->touch(dirname($target));
  345. return true;
  346. }
  347. public function getMimeType(string $path): string|false {
  348. $path = $this->normalizePath($path);
  349. return parent::getMimeType($path);
  350. }
  351. public function touch(string $path, ?int $mtime = null): bool {
  352. if (is_null($mtime)) {
  353. $mtime = time();
  354. }
  355. $path = $this->normalizePath($path);
  356. $dirName = dirname($path);
  357. $parentExists = $this->is_dir($dirName);
  358. if (!$parentExists) {
  359. return false;
  360. }
  361. $stat = $this->stat($path);
  362. if (is_array($stat)) {
  363. // update existing mtime in db
  364. $stat['mtime'] = $mtime;
  365. $this->getCache()->update($stat['fileid'], $stat);
  366. } else {
  367. try {
  368. //create a empty file, need to have at least on char to make it
  369. // work with all object storage implementations
  370. $this->file_put_contents($path, ' ');
  371. $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
  372. $stat = [
  373. 'etag' => $this->getETag($path),
  374. 'mimetype' => $mimeType,
  375. 'size' => 0,
  376. 'mtime' => $mtime,
  377. 'storage_mtime' => $mtime,
  378. 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
  379. ];
  380. $this->getCache()->put($path, $stat);
  381. } catch (\Exception $ex) {
  382. $this->logger->error(
  383. 'Could not create object for ' . $path,
  384. [
  385. 'app' => 'objectstore',
  386. 'exception' => $ex,
  387. ]
  388. );
  389. throw $ex;
  390. }
  391. }
  392. return true;
  393. }
  394. public function writeBack(string $tmpFile, string $path) {
  395. $size = filesize($tmpFile);
  396. $this->writeStream($path, fopen($tmpFile, 'r'), $size);
  397. }
  398. public function hasUpdated(string $path, int $time): bool {
  399. return false;
  400. }
  401. public function needsPartFile(): bool {
  402. return false;
  403. }
  404. public function file_put_contents(string $path, mixed $data): int {
  405. $fh = fopen('php://temp', 'w+');
  406. fwrite($fh, $data);
  407. rewind($fh);
  408. return $this->writeStream($path, $fh, strlen($data));
  409. }
  410. public function writeStream(string $path, $stream, ?int $size = null): int {
  411. if ($size === null) {
  412. $stats = fstat($stream);
  413. if (is_array($stats) && isset($stats['size'])) {
  414. $size = $stats['size'];
  415. }
  416. }
  417. $stat = $this->stat($path);
  418. if (empty($stat)) {
  419. // create new file
  420. $stat = [
  421. 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
  422. ];
  423. }
  424. // update stat with new data
  425. $mTime = time();
  426. $stat['size'] = (int)$size;
  427. $stat['mtime'] = $mTime;
  428. $stat['storage_mtime'] = $mTime;
  429. $mimetypeDetector = \OC::$server->getMimeTypeDetector();
  430. $mimetype = $mimetypeDetector->detectPath($path);
  431. $stat['mimetype'] = $mimetype;
  432. $stat['etag'] = $this->getETag($path);
  433. $stat['checksum'] = '';
  434. $exists = $this->getCache()->inCache($path);
  435. $uploadPath = $exists ? $path : $path . '.part';
  436. if ($exists) {
  437. $fileId = $stat['fileid'];
  438. } else {
  439. $parent = $this->normalizePath(dirname($path));
  440. if (!$this->is_dir($parent)) {
  441. throw new \InvalidArgumentException("trying to upload a file ($path) inside a non-directory ($parent)");
  442. }
  443. $fileId = $this->getCache()->put($uploadPath, $stat);
  444. }
  445. $urn = $this->getURN($fileId);
  446. try {
  447. //upload to object storage
  448. if ($size === null) {
  449. $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) {
  450. $this->getCache()->update($fileId, [
  451. 'size' => $writtenSize,
  452. ]);
  453. $size = $writtenSize;
  454. });
  455. $this->objectStore->writeObject($urn, $countStream, $mimetype);
  456. if (is_resource($countStream)) {
  457. fclose($countStream);
  458. }
  459. $stat['size'] = $size;
  460. } else {
  461. $this->objectStore->writeObject($urn, $stream, $mimetype);
  462. if (is_resource($stream)) {
  463. fclose($stream);
  464. }
  465. }
  466. } catch (\Exception $ex) {
  467. if (!$exists) {
  468. /*
  469. * Only remove the entry if we are dealing with a new file.
  470. * Else people lose access to existing files
  471. */
  472. $this->getCache()->remove($uploadPath);
  473. $this->logger->error(
  474. 'Could not create object ' . $urn . ' for ' . $path,
  475. [
  476. 'app' => 'objectstore',
  477. 'exception' => $ex,
  478. ]
  479. );
  480. } else {
  481. $this->logger->error(
  482. 'Could not update object ' . $urn . ' for ' . $path,
  483. [
  484. 'app' => 'objectstore',
  485. 'exception' => $ex,
  486. ]
  487. );
  488. }
  489. throw $ex; // make this bubble up
  490. }
  491. if ($exists) {
  492. // Always update the unencrypted size, for encryption the Encryption wrapper will update this afterwards anyways
  493. $stat['unencrypted_size'] = $stat['size'];
  494. $this->getCache()->update($fileId, $stat);
  495. } else {
  496. if (!$this->validateWrites || $this->objectStore->objectExists($urn)) {
  497. $this->getCache()->move($uploadPath, $path);
  498. } else {
  499. $this->getCache()->remove($uploadPath);
  500. throw new \Exception("Object not found after writing (urn: $urn, path: $path)", 404);
  501. }
  502. }
  503. return $size;
  504. }
  505. public function getObjectStore(): IObjectStore {
  506. return $this->objectStore;
  507. }
  508. public function copyFromStorage(
  509. IStorage $sourceStorage,
  510. string $sourceInternalPath,
  511. string $targetInternalPath,
  512. bool $preserveMtime = false,
  513. ): bool {
  514. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  515. /** @var ObjectStoreStorage $sourceStorage */
  516. if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) {
  517. /** @var CacheEntry $sourceEntry */
  518. $sourceEntry = $sourceStorage->getCache()->get($sourceInternalPath);
  519. $sourceEntryData = $sourceEntry->getData();
  520. // $sourceEntry['permissions'] here is the permissions from the jailed storage for the current
  521. // user. Instead we use $sourceEntryData['scan_permissions'] that are the permissions from the
  522. // unjailed storage.
  523. if (is_array($sourceEntryData) && array_key_exists('scan_permissions', $sourceEntryData)) {
  524. $sourceEntry['permissions'] = $sourceEntryData['scan_permissions'];
  525. }
  526. $this->copyInner($sourceStorage->getCache(), $sourceEntry, $targetInternalPath);
  527. return true;
  528. }
  529. }
  530. return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
  531. }
  532. public function moveFromStorage(IStorage $sourceStorage, string $sourceInternalPath, string $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool {
  533. $sourceCache = $sourceStorage->getCache();
  534. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class) && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) {
  535. $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath);
  536. // Do not import any data when source and target bucket are identical.
  537. return true;
  538. }
  539. if (!$sourceCacheEntry) {
  540. $sourceCacheEntry = $sourceCache->get($sourceInternalPath);
  541. }
  542. $this->copyObjects($sourceStorage, $sourceCache, $sourceCacheEntry);
  543. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  544. /** @var ObjectStoreStorage $sourceStorage */
  545. $sourceStorage->setPreserveCacheOnDelete(true);
  546. }
  547. if ($sourceCacheEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  548. $sourceStorage->rmdir($sourceInternalPath);
  549. } else {
  550. $sourceStorage->unlink($sourceInternalPath);
  551. }
  552. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  553. /** @var ObjectStoreStorage $sourceStorage */
  554. $sourceStorage->setPreserveCacheOnDelete(false);
  555. }
  556. $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath);
  557. return true;
  558. }
  559. /**
  560. * Copy the object(s) of a file or folder into this storage, without touching the cache
  561. */
  562. private function copyObjects(IStorage $sourceStorage, ICache $sourceCache, ICacheEntry $sourceCacheEntry) {
  563. $copiedFiles = [];
  564. try {
  565. foreach ($this->getAllChildObjects($sourceCache, $sourceCacheEntry) as $file) {
  566. $sourceStream = $sourceStorage->fopen($file->getPath(), 'r');
  567. if (!$sourceStream) {
  568. throw new \Exception("Failed to open source file {$file->getPath()} ({$file->getId()})");
  569. }
  570. $this->objectStore->writeObject($this->getURN($file->getId()), $sourceStream, $file->getMimeType());
  571. if (is_resource($sourceStream)) {
  572. fclose($sourceStream);
  573. }
  574. $copiedFiles[] = $file->getId();
  575. }
  576. } catch (\Exception $e) {
  577. foreach ($copiedFiles as $fileId) {
  578. try {
  579. $this->objectStore->deleteObject($this->getURN($fileId));
  580. } catch (\Exception $e) {
  581. // ignore
  582. }
  583. }
  584. throw $e;
  585. }
  586. }
  587. /**
  588. * @return \Iterator<ICacheEntry>
  589. */
  590. private function getAllChildObjects(ICache $cache, ICacheEntry $entry): \Iterator {
  591. if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
  592. foreach ($cache->getFolderContentsById($entry->getId()) as $child) {
  593. yield from $this->getAllChildObjects($cache, $child);
  594. }
  595. } else {
  596. yield $entry;
  597. }
  598. }
  599. public function copy(string $source, string $target): bool {
  600. $source = $this->normalizePath($source);
  601. $target = $this->normalizePath($target);
  602. $cache = $this->getCache();
  603. $sourceEntry = $cache->get($source);
  604. if (!$sourceEntry) {
  605. throw new NotFoundException('Source object not found');
  606. }
  607. $this->copyInner($cache, $sourceEntry, $target);
  608. return true;
  609. }
  610. private function copyInner(ICache $sourceCache, ICacheEntry $sourceEntry, string $to) {
  611. $cache = $this->getCache();
  612. if ($sourceEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
  613. if ($cache->inCache($to)) {
  614. $cache->remove($to);
  615. }
  616. $this->mkdir($to);
  617. foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) {
  618. $this->copyInner($sourceCache, $child, $to . '/' . $child->getName());
  619. }
  620. } else {
  621. $this->copyFile($sourceEntry, $to);
  622. }
  623. }
  624. private function copyFile(ICacheEntry $sourceEntry, string $to) {
  625. $cache = $this->getCache();
  626. $sourceUrn = $this->getURN($sourceEntry->getId());
  627. if (!$cache instanceof Cache) {
  628. throw new \Exception('Invalid source cache for object store copy');
  629. }
  630. $targetId = $cache->copyFromCache($cache, $sourceEntry, $to);
  631. $targetUrn = $this->getURN($targetId);
  632. try {
  633. $this->objectStore->copyObject($sourceUrn, $targetUrn);
  634. if ($this->handleCopiesAsOwned) {
  635. // Copied the file thus we gain all permissions as we are the owner now ! warning while this aligns with local storage it should not be used and instead fix local storage !
  636. $cache->update($targetId, ['permissions' => \OCP\Constants::PERMISSION_ALL]);
  637. }
  638. } catch (\Exception $e) {
  639. $cache->remove($to);
  640. throw $e;
  641. }
  642. }
  643. public function startChunkedWrite(string $targetPath): string {
  644. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  645. throw new GenericFileException('Object store does not support multipart upload');
  646. }
  647. $cacheEntry = $this->getCache()->get($targetPath);
  648. $urn = $this->getURN($cacheEntry->getId());
  649. return $this->objectStore->initiateMultipartUpload($urn);
  650. }
  651. /**
  652. * @throws GenericFileException
  653. */
  654. public function putChunkedWritePart(
  655. string $targetPath,
  656. string $writeToken,
  657. string $chunkId,
  658. $data,
  659. $size = null,
  660. ): ?array {
  661. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  662. throw new GenericFileException('Object store does not support multipart upload');
  663. }
  664. $cacheEntry = $this->getCache()->get($targetPath);
  665. $urn = $this->getURN($cacheEntry->getId());
  666. $result = $this->objectStore->uploadMultipartPart($urn, $writeToken, (int)$chunkId, $data, $size);
  667. $parts[$chunkId] = [
  668. 'PartNumber' => $chunkId,
  669. 'ETag' => trim($result->get('ETag'), '"'),
  670. ];
  671. return $parts[$chunkId];
  672. }
  673. public function completeChunkedWrite(string $targetPath, string $writeToken): int {
  674. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  675. throw new GenericFileException('Object store does not support multipart upload');
  676. }
  677. $cacheEntry = $this->getCache()->get($targetPath);
  678. $urn = $this->getURN($cacheEntry->getId());
  679. $parts = $this->objectStore->getMultipartUploads($urn, $writeToken);
  680. $sortedParts = array_values($parts);
  681. sort($sortedParts);
  682. try {
  683. $size = $this->objectStore->completeMultipartUpload($urn, $writeToken, $sortedParts);
  684. $stat = $this->stat($targetPath);
  685. $mtime = time();
  686. if (is_array($stat)) {
  687. $stat['size'] = $size;
  688. $stat['mtime'] = $mtime;
  689. $stat['mimetype'] = $this->getMimeType($targetPath);
  690. $this->getCache()->update($stat['fileid'], $stat);
  691. }
  692. } catch (S3MultipartUploadException|S3Exception $e) {
  693. $this->objectStore->abortMultipartUpload($urn, $writeToken);
  694. $this->logger->error(
  695. 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken,
  696. [
  697. 'app' => 'objectstore',
  698. 'exception' => $e,
  699. ]
  700. );
  701. throw new GenericFileException('Could not write chunked file');
  702. }
  703. return $size;
  704. }
  705. public function cancelChunkedWrite(string $targetPath, string $writeToken): void {
  706. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  707. throw new GenericFileException('Object store does not support multipart upload');
  708. }
  709. $cacheEntry = $this->getCache()->get($targetPath);
  710. $urn = $this->getURN($cacheEntry->getId());
  711. $this->objectStore->abortMultipartUpload($urn, $writeToken);
  712. }
  713. public function setPreserveCacheOnDelete(bool $preserve) {
  714. $this->preserveCacheItemsOnDelete = $preserve;
  715. }
  716. }