1
0

ObjectStoreStorage.php 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823
  1. <?php
  2. /**
  3. * @copyright Copyright (c) 2016, ownCloud, Inc.
  4. *
  5. * @author Bjoern Schiessle <bjoern@schiessle.org>
  6. * @author Christoph Wurst <christoph@winzerhof-wurst.at>
  7. * @author Joas Schilling <coding@schilljs.com>
  8. * @author Jörn Friedrich Dreyer <jfd@butonic.de>
  9. * @author Marcel Klehr <mklehr@gmx.net>
  10. * @author Morris Jobke <hey@morrisjobke.de>
  11. * @author Robin Appelman <robin@icewind.nl>
  12. * @author Roeland Jago Douma <roeland@famdouma.nl>
  13. * @author Tigran Mkrtchyan <tigran.mkrtchyan@desy.de>
  14. *
  15. * @license AGPL-3.0
  16. *
  17. * This code is free software: you can redistribute it and/or modify
  18. * it under the terms of the GNU Affero General Public License, version 3,
  19. * as published by the Free Software Foundation.
  20. *
  21. * This program is distributed in the hope that it will be useful,
  22. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  23. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  24. * GNU Affero General Public License for more details.
  25. *
  26. * You should have received a copy of the GNU Affero General Public License, version 3,
  27. * along with this program. If not, see <http://www.gnu.org/licenses/>
  28. *
  29. */
  30. namespace OC\Files\ObjectStore;
  31. use Aws\S3\Exception\S3Exception;
  32. use Aws\S3\Exception\S3MultipartUploadException;
  33. use Icewind\Streams\CallbackWrapper;
  34. use Icewind\Streams\CountWrapper;
  35. use Icewind\Streams\IteratorDirectory;
  36. use OC\Files\Cache\Cache;
  37. use OC\Files\Cache\CacheEntry;
  38. use OC\Files\Storage\PolyFill\CopyDirectory;
  39. use OCP\Files\Cache\ICache;
  40. use OCP\Files\Cache\ICacheEntry;
  41. use OCP\Files\FileInfo;
  42. use OCP\Files\GenericFileException;
  43. use OCP\Files\NotFoundException;
  44. use OCP\Files\ObjectStore\IObjectStore;
  45. use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
  46. use OCP\Files\Storage\IChunkedFileWrite;
  47. use OCP\Files\Storage\IStorage;
  48. class ObjectStoreStorage extends \OC\Files\Storage\Common implements IChunkedFileWrite {
  49. use CopyDirectory;
  50. /**
  51. * @var \OCP\Files\ObjectStore\IObjectStore $objectStore
  52. */
  53. protected $objectStore;
  54. /**
  55. * @var string $id
  56. */
  57. protected $id;
  58. /**
  59. * @var \OC\User\User $user
  60. */
  61. protected $user;
  62. private $objectPrefix = 'urn:oid:';
  63. private $logger;
  64. private bool $handleCopiesAsOwned;
  65. private bool $preserveCacheItemsOnDelete = false;
  66. /** @var bool */
  67. protected $validateWrites = true;
  68. public function __construct($params) {
  69. if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) {
  70. $this->objectStore = $params['objectstore'];
  71. } else {
  72. throw new \Exception('missing IObjectStore instance');
  73. }
  74. if (isset($params['storageid'])) {
  75. $this->id = 'object::store:' . $params['storageid'];
  76. } else {
  77. $this->id = 'object::store:' . $this->objectStore->getStorageId();
  78. }
  79. if (isset($params['objectPrefix'])) {
  80. $this->objectPrefix = $params['objectPrefix'];
  81. }
  82. if (isset($params['validateWrites'])) {
  83. $this->validateWrites = (bool)$params['validateWrites'];
  84. }
  85. $this->handleCopiesAsOwned = (bool)($params['handleCopiesAsOwned'] ?? false);
  86. $this->logger = \OC::$server->getLogger();
  87. }
  88. public function mkdir($path, bool $force = false) {
  89. $path = $this->normalizePath($path);
  90. if (!$force && $this->file_exists($path)) {
  91. $this->logger->warning("Tried to create an object store folder that already exists: $path");
  92. return false;
  93. }
  94. $mTime = time();
  95. $data = [
  96. 'mimetype' => 'httpd/unix-directory',
  97. 'size' => 0,
  98. 'mtime' => $mTime,
  99. 'storage_mtime' => $mTime,
  100. 'permissions' => \OCP\Constants::PERMISSION_ALL,
  101. ];
  102. if ($path === '') {
  103. //create root on the fly
  104. $data['etag'] = $this->getETag('');
  105. $this->getCache()->put('', $data);
  106. return true;
  107. } else {
  108. // if parent does not exist, create it
  109. $parent = $this->normalizePath(dirname($path));
  110. $parentType = $this->filetype($parent);
  111. if ($parentType === false) {
  112. if (!$this->mkdir($parent)) {
  113. // something went wrong
  114. $this->logger->warning("Parent folder ($parent) doesn't exist and couldn't be created");
  115. return false;
  116. }
  117. } elseif ($parentType === 'file') {
  118. // parent is a file
  119. $this->logger->warning("Parent ($parent) is a file");
  120. return false;
  121. }
  122. // finally create the new dir
  123. $mTime = time(); // update mtime
  124. $data['mtime'] = $mTime;
  125. $data['storage_mtime'] = $mTime;
  126. $data['etag'] = $this->getETag($path);
  127. $this->getCache()->put($path, $data);
  128. return true;
  129. }
  130. }
  131. /**
  132. * @param string $path
  133. * @return string
  134. */
  135. private function normalizePath($path) {
  136. $path = trim($path, '/');
  137. //FIXME why do we sometimes get a path like 'files//username'?
  138. $path = str_replace('//', '/', $path);
  139. // dirname('/folder') returns '.' but internally (in the cache) we store the root as ''
  140. if (!$path || $path === '.') {
  141. $path = '';
  142. }
  143. return $path;
  144. }
  145. /**
  146. * Object Stores use a NoopScanner because metadata is directly stored in
  147. * the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
  148. *
  149. * @param string $path
  150. * @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner
  151. * @return \OC\Files\ObjectStore\ObjectStoreScanner
  152. */
  153. public function getScanner($path = '', $storage = null) {
  154. if (!$storage) {
  155. $storage = $this;
  156. }
  157. if (!isset($this->scanner)) {
  158. $this->scanner = new ObjectStoreScanner($storage);
  159. }
  160. return $this->scanner;
  161. }
  162. public function getId() {
  163. return $this->id;
  164. }
  165. public function rmdir($path) {
  166. $path = $this->normalizePath($path);
  167. $entry = $this->getCache()->get($path);
  168. if (!$entry || $entry->getMimeType() !== ICacheEntry::DIRECTORY_MIMETYPE) {
  169. return false;
  170. }
  171. return $this->rmObjects($entry);
  172. }
  173. private function rmObjects(ICacheEntry $entry): bool {
  174. $children = $this->getCache()->getFolderContentsById($entry->getId());
  175. foreach ($children as $child) {
  176. if ($child->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  177. if (!$this->rmObjects($child)) {
  178. return false;
  179. }
  180. } else {
  181. if (!$this->rmObject($child)) {
  182. return false;
  183. }
  184. }
  185. }
  186. if (!$this->preserveCacheItemsOnDelete) {
  187. $this->getCache()->remove($entry->getPath());
  188. }
  189. return true;
  190. }
  191. public function unlink($path) {
  192. $path = $this->normalizePath($path);
  193. $entry = $this->getCache()->get($path);
  194. if ($entry instanceof ICacheEntry) {
  195. if ($entry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  196. return $this->rmObjects($entry);
  197. } else {
  198. return $this->rmObject($entry);
  199. }
  200. }
  201. return false;
  202. }
  203. public function rmObject(ICacheEntry $entry): bool {
  204. try {
  205. $this->objectStore->deleteObject($this->getURN($entry->getId()));
  206. } catch (\Exception $ex) {
  207. if ($ex->getCode() !== 404) {
  208. $this->logger->logException($ex, [
  209. 'app' => 'objectstore',
  210. 'message' => 'Could not delete object ' . $this->getURN($entry->getId()) . ' for ' . $entry->getPath(),
  211. ]);
  212. return false;
  213. }
  214. //removing from cache is ok as it does not exist in the objectstore anyway
  215. }
  216. if (!$this->preserveCacheItemsOnDelete) {
  217. $this->getCache()->remove($entry->getPath());
  218. }
  219. return true;
  220. }
  221. public function stat($path) {
  222. $path = $this->normalizePath($path);
  223. $cacheEntry = $this->getCache()->get($path);
  224. if ($cacheEntry instanceof CacheEntry) {
  225. return $cacheEntry->getData();
  226. } else {
  227. if ($path === '') {
  228. $this->mkdir('', true);
  229. $cacheEntry = $this->getCache()->get($path);
  230. if ($cacheEntry instanceof CacheEntry) {
  231. return $cacheEntry->getData();
  232. }
  233. }
  234. return false;
  235. }
  236. }
  237. public function getPermissions($path) {
  238. $stat = $this->stat($path);
  239. if (is_array($stat) && isset($stat['permissions'])) {
  240. return $stat['permissions'];
  241. }
  242. return parent::getPermissions($path);
  243. }
  244. /**
  245. * Override this method if you need a different unique resource identifier for your object storage implementation.
  246. * The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
  247. * You may need a mapping table to store your URN if it cannot be generated from the fileid.
  248. *
  249. * @param int $fileId the fileid
  250. * @return null|string the unified resource name used to identify the object
  251. */
  252. public function getURN($fileId) {
  253. if (is_numeric($fileId)) {
  254. return $this->objectPrefix . $fileId;
  255. }
  256. return null;
  257. }
  258. public function opendir($path) {
  259. $path = $this->normalizePath($path);
  260. try {
  261. $files = [];
  262. $folderContents = $this->getCache()->getFolderContents($path);
  263. foreach ($folderContents as $file) {
  264. $files[] = $file['name'];
  265. }
  266. return IteratorDirectory::wrap($files);
  267. } catch (\Exception $e) {
  268. $this->logger->logException($e);
  269. return false;
  270. }
  271. }
  272. public function filetype($path) {
  273. $path = $this->normalizePath($path);
  274. $stat = $this->stat($path);
  275. if ($stat) {
  276. if ($stat['mimetype'] === 'httpd/unix-directory') {
  277. return 'dir';
  278. }
  279. return 'file';
  280. } else {
  281. return false;
  282. }
  283. }
  284. public function fopen($path, $mode) {
  285. $path = $this->normalizePath($path);
  286. if (strrpos($path, '.') !== false) {
  287. $ext = substr($path, strrpos($path, '.'));
  288. } else {
  289. $ext = '';
  290. }
  291. switch ($mode) {
  292. case 'r':
  293. case 'rb':
  294. $stat = $this->stat($path);
  295. if (is_array($stat)) {
  296. $filesize = $stat['size'] ?? 0;
  297. // Reading 0 sized files is a waste of time
  298. if ($filesize === 0) {
  299. return fopen('php://memory', $mode);
  300. }
  301. try {
  302. $handle = $this->objectStore->readObject($this->getURN($stat['fileid']));
  303. if ($handle === false) {
  304. return false; // keep backward compatibility
  305. }
  306. $streamStat = fstat($handle);
  307. $actualSize = $streamStat['size'] ?? -1;
  308. if ($actualSize > -1 && $actualSize !== $filesize) {
  309. $this->getCache()->update((int)$stat['fileid'], ['size' => $actualSize]);
  310. }
  311. return $handle;
  312. } catch (NotFoundException $e) {
  313. $this->logger->logException($e, [
  314. 'app' => 'objectstore',
  315. 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
  316. ]);
  317. throw $e;
  318. } catch (\Exception $ex) {
  319. $this->logger->logException($ex, [
  320. 'app' => 'objectstore',
  321. 'message' => 'Could not get object ' . $this->getURN($stat['fileid']) . ' for file ' . $path,
  322. ]);
  323. return false;
  324. }
  325. } else {
  326. return false;
  327. }
  328. // no break
  329. case 'w':
  330. case 'wb':
  331. case 'w+':
  332. case 'wb+':
  333. $dirName = dirname($path);
  334. $parentExists = $this->is_dir($dirName);
  335. if (!$parentExists) {
  336. return false;
  337. }
  338. $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
  339. $handle = fopen($tmpFile, $mode);
  340. return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
  341. $this->writeBack($tmpFile, $path);
  342. unlink($tmpFile);
  343. });
  344. case 'a':
  345. case 'ab':
  346. case 'r+':
  347. case 'a+':
  348. case 'x':
  349. case 'x+':
  350. case 'c':
  351. case 'c+':
  352. $tmpFile = \OC::$server->getTempManager()->getTemporaryFile($ext);
  353. if ($this->file_exists($path)) {
  354. $source = $this->fopen($path, 'r');
  355. file_put_contents($tmpFile, $source);
  356. }
  357. $handle = fopen($tmpFile, $mode);
  358. return CallbackWrapper::wrap($handle, null, null, function () use ($path, $tmpFile) {
  359. $this->writeBack($tmpFile, $path);
  360. unlink($tmpFile);
  361. });
  362. }
  363. return false;
  364. }
  365. public function file_exists($path) {
  366. $path = $this->normalizePath($path);
  367. return (bool)$this->stat($path);
  368. }
  369. public function rename($source, $target) {
  370. $source = $this->normalizePath($source);
  371. $target = $this->normalizePath($target);
  372. $this->remove($target);
  373. $this->getCache()->move($source, $target);
  374. $this->touch(dirname($target));
  375. return true;
  376. }
  377. public function getMimeType($path) {
  378. $path = $this->normalizePath($path);
  379. return parent::getMimeType($path);
  380. }
  381. public function touch($path, $mtime = null) {
  382. if (is_null($mtime)) {
  383. $mtime = time();
  384. }
  385. $path = $this->normalizePath($path);
  386. $dirName = dirname($path);
  387. $parentExists = $this->is_dir($dirName);
  388. if (!$parentExists) {
  389. return false;
  390. }
  391. $stat = $this->stat($path);
  392. if (is_array($stat)) {
  393. // update existing mtime in db
  394. $stat['mtime'] = $mtime;
  395. $this->getCache()->update($stat['fileid'], $stat);
  396. } else {
  397. try {
  398. //create a empty file, need to have at least on char to make it
  399. // work with all object storage implementations
  400. $this->file_put_contents($path, ' ');
  401. $mimeType = \OC::$server->getMimeTypeDetector()->detectPath($path);
  402. $stat = [
  403. 'etag' => $this->getETag($path),
  404. 'mimetype' => $mimeType,
  405. 'size' => 0,
  406. 'mtime' => $mtime,
  407. 'storage_mtime' => $mtime,
  408. 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
  409. ];
  410. $this->getCache()->put($path, $stat);
  411. } catch (\Exception $ex) {
  412. $this->logger->logException($ex, [
  413. 'app' => 'objectstore',
  414. 'message' => 'Could not create object for ' . $path,
  415. ]);
  416. throw $ex;
  417. }
  418. }
  419. return true;
  420. }
  421. public function writeBack($tmpFile, $path) {
  422. $size = filesize($tmpFile);
  423. $this->writeStream($path, fopen($tmpFile, 'r'), $size);
  424. }
  425. /**
  426. * external changes are not supported, exclusive access to the object storage is assumed
  427. *
  428. * @param string $path
  429. * @param int $time
  430. * @return false
  431. */
  432. public function hasUpdated($path, $time) {
  433. return false;
  434. }
  435. public function needsPartFile() {
  436. return false;
  437. }
  438. public function file_put_contents($path, $data) {
  439. $fh = fopen('php://temp', 'w+');
  440. fwrite($fh, $data);
  441. rewind($fh);
  442. return $this->writeStream($path, $fh, strlen($data));
  443. }
  444. public function writeStream(string $path, $stream, int $size = null): int {
  445. $stat = $this->stat($path);
  446. if (empty($stat)) {
  447. // create new file
  448. $stat = [
  449. 'permissions' => \OCP\Constants::PERMISSION_ALL - \OCP\Constants::PERMISSION_CREATE,
  450. ];
  451. }
  452. // update stat with new data
  453. $mTime = time();
  454. $stat['size'] = (int)$size;
  455. $stat['mtime'] = $mTime;
  456. $stat['storage_mtime'] = $mTime;
  457. $mimetypeDetector = \OC::$server->getMimeTypeDetector();
  458. $mimetype = $mimetypeDetector->detectPath($path);
  459. $stat['mimetype'] = $mimetype;
  460. $stat['etag'] = $this->getETag($path);
  461. $stat['checksum'] = '';
  462. $exists = $this->getCache()->inCache($path);
  463. $uploadPath = $exists ? $path : $path . '.part';
  464. if ($exists) {
  465. $fileId = $stat['fileid'];
  466. } else {
  467. $parent = $this->normalizePath(dirname($path));
  468. if (!$this->is_dir($parent)) {
  469. throw new \InvalidArgumentException("trying to upload a file ($path) inside a non-directory ($parent)");
  470. }
  471. $fileId = $this->getCache()->put($uploadPath, $stat);
  472. }
  473. $urn = $this->getURN($fileId);
  474. try {
  475. //upload to object storage
  476. if ($size === null) {
  477. $countStream = CountWrapper::wrap($stream, function ($writtenSize) use ($fileId, &$size) {
  478. $this->getCache()->update($fileId, [
  479. 'size' => $writtenSize,
  480. ]);
  481. $size = $writtenSize;
  482. });
  483. $this->objectStore->writeObject($urn, $countStream, $mimetype);
  484. if (is_resource($countStream)) {
  485. fclose($countStream);
  486. }
  487. $stat['size'] = $size;
  488. } else {
  489. $this->objectStore->writeObject($urn, $stream, $mimetype);
  490. if (is_resource($stream)) {
  491. fclose($stream);
  492. }
  493. }
  494. } catch (\Exception $ex) {
  495. if (!$exists) {
  496. /*
  497. * Only remove the entry if we are dealing with a new file.
  498. * Else people lose access to existing files
  499. */
  500. $this->getCache()->remove($uploadPath);
  501. $this->logger->logException($ex, [
  502. 'app' => 'objectstore',
  503. 'message' => 'Could not create object ' . $urn . ' for ' . $path,
  504. ]);
  505. } else {
  506. $this->logger->logException($ex, [
  507. 'app' => 'objectstore',
  508. 'message' => 'Could not update object ' . $urn . ' for ' . $path,
  509. ]);
  510. }
  511. throw $ex; // make this bubble up
  512. }
  513. if ($exists) {
  514. // Always update the unencrypted size, for encryption the Encryption wrapper will update this afterwards anyways
  515. $stat['unencrypted_size'] = $stat['size'];
  516. $this->getCache()->update($fileId, $stat);
  517. } else {
  518. if (!$this->validateWrites || $this->objectStore->objectExists($urn)) {
  519. $this->getCache()->move($uploadPath, $path);
  520. } else {
  521. $this->getCache()->remove($uploadPath);
  522. throw new \Exception("Object not found after writing (urn: $urn, path: $path)", 404);
  523. }
  524. }
  525. return $size;
  526. }
  527. public function getObjectStore(): IObjectStore {
  528. return $this->objectStore;
  529. }
  530. public function copyFromStorage(
  531. IStorage $sourceStorage,
  532. $sourceInternalPath,
  533. $targetInternalPath,
  534. $preserveMtime = false
  535. ) {
  536. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  537. /** @var ObjectStoreStorage $sourceStorage */
  538. if ($sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) {
  539. /** @var CacheEntry $sourceEntry */
  540. $sourceEntry = $sourceStorage->getCache()->get($sourceInternalPath);
  541. $sourceEntryData = $sourceEntry->getData();
  542. // $sourceEntry['permissions'] here is the permissions from the jailed storage for the current
  543. // user. Instead we use $sourceEntryData['scan_permissions'] that are the permissions from the
  544. // unjailed storage.
  545. if (is_array($sourceEntryData) && array_key_exists('scan_permissions', $sourceEntryData)) {
  546. $sourceEntry['permissions'] = $sourceEntryData['scan_permissions'];
  547. }
  548. $this->copyInner($sourceStorage->getCache(), $sourceEntry, $targetInternalPath);
  549. return true;
  550. }
  551. }
  552. return parent::copyFromStorage($sourceStorage, $sourceInternalPath, $targetInternalPath);
  553. }
  554. public function moveFromStorage(IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, ?ICacheEntry $sourceCacheEntry = null): bool {
  555. $sourceCache = $sourceStorage->getCache();
  556. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class) && $sourceStorage->getObjectStore()->getStorageId() === $this->getObjectStore()->getStorageId()) {
  557. $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath);
  558. // Do not import any data when source and target bucket are identical.
  559. return true;
  560. }
  561. if (!$sourceCacheEntry) {
  562. $sourceCacheEntry = $sourceCache->get($sourceInternalPath);
  563. }
  564. $this->copyObjects($sourceStorage, $sourceCache, $sourceCacheEntry);
  565. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  566. /** @var ObjectStoreStorage $sourceStorage */
  567. $sourceStorage->setPreserveCacheOnDelete(true);
  568. }
  569. if ($sourceCacheEntry->getMimeType() === ICacheEntry::DIRECTORY_MIMETYPE) {
  570. $sourceStorage->rmdir($sourceInternalPath);
  571. } else {
  572. $sourceStorage->unlink($sourceInternalPath);
  573. }
  574. if ($sourceStorage->instanceOfStorage(ObjectStoreStorage::class)) {
  575. /** @var ObjectStoreStorage $sourceStorage */
  576. $sourceStorage->setPreserveCacheOnDelete(false);
  577. }
  578. $this->getCache()->moveFromCache($sourceCache, $sourceInternalPath, $targetInternalPath);
  579. return true;
  580. }
  581. /**
  582. * Copy the object(s) of a file or folder into this storage, without touching the cache
  583. */
  584. private function copyObjects(IStorage $sourceStorage, ICache $sourceCache, ICacheEntry $sourceCacheEntry) {
  585. $copiedFiles = [];
  586. try {
  587. foreach ($this->getAllChildObjects($sourceCache, $sourceCacheEntry) as $file) {
  588. $sourceStream = $sourceStorage->fopen($file->getPath(), 'r');
  589. if (!$sourceStream) {
  590. throw new \Exception("Failed to open source file {$file->getPath()} ({$file->getId()})");
  591. }
  592. $this->objectStore->writeObject($this->getURN($file->getId()), $sourceStream, $file->getMimeType());
  593. if (is_resource($sourceStream)) {
  594. fclose($sourceStream);
  595. }
  596. $copiedFiles[] = $file->getId();
  597. }
  598. } catch (\Exception $e) {
  599. foreach ($copiedFiles as $fileId) {
  600. try {
  601. $this->objectStore->deleteObject($this->getURN($fileId));
  602. } catch (\Exception $e) {
  603. // ignore
  604. }
  605. }
  606. throw $e;
  607. }
  608. }
  609. /**
  610. * @return \Iterator<ICacheEntry>
  611. */
  612. private function getAllChildObjects(ICache $cache, ICacheEntry $entry): \Iterator {
  613. if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
  614. foreach ($cache->getFolderContentsById($entry->getId()) as $child) {
  615. yield from $this->getAllChildObjects($cache, $child);
  616. }
  617. } else {
  618. yield $entry;
  619. }
  620. }
  621. public function copy($source, $target) {
  622. $source = $this->normalizePath($source);
  623. $target = $this->normalizePath($target);
  624. $cache = $this->getCache();
  625. $sourceEntry = $cache->get($source);
  626. if (!$sourceEntry) {
  627. throw new NotFoundException('Source object not found');
  628. }
  629. $this->copyInner($cache, $sourceEntry, $target);
  630. return true;
  631. }
  632. private function copyInner(ICache $sourceCache, ICacheEntry $sourceEntry, string $to) {
  633. $cache = $this->getCache();
  634. if ($sourceEntry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
  635. if ($cache->inCache($to)) {
  636. $cache->remove($to);
  637. }
  638. $this->mkdir($to);
  639. foreach ($sourceCache->getFolderContentsById($sourceEntry->getId()) as $child) {
  640. $this->copyInner($sourceCache, $child, $to . '/' . $child->getName());
  641. }
  642. } else {
  643. $this->copyFile($sourceEntry, $to);
  644. }
  645. }
  646. private function copyFile(ICacheEntry $sourceEntry, string $to) {
  647. $cache = $this->getCache();
  648. $sourceUrn = $this->getURN($sourceEntry->getId());
  649. if (!$cache instanceof Cache) {
  650. throw new \Exception("Invalid source cache for object store copy");
  651. }
  652. $targetId = $cache->copyFromCache($cache, $sourceEntry, $to);
  653. $targetUrn = $this->getURN($targetId);
  654. try {
  655. $this->objectStore->copyObject($sourceUrn, $targetUrn);
  656. if ($this->handleCopiesAsOwned) {
  657. // Copied the file thus we gain all permissions as we are the owner now ! warning while this aligns with local storage it should not be used and instead fix local storage !
  658. $cache->update($targetId, ['permissions' => \OCP\Constants::PERMISSION_ALL]);
  659. }
  660. } catch (\Exception $e) {
  661. $cache->remove($to);
  662. throw $e;
  663. }
  664. }
  665. public function startChunkedWrite(string $targetPath): string {
  666. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  667. throw new GenericFileException('Object store does not support multipart upload');
  668. }
  669. $cacheEntry = $this->getCache()->get($targetPath);
  670. $urn = $this->getURN($cacheEntry->getId());
  671. return $this->objectStore->initiateMultipartUpload($urn);
  672. }
  673. /**
  674. *
  675. * @throws GenericFileException
  676. */
  677. public function putChunkedWritePart(
  678. string $targetPath,
  679. string $writeToken,
  680. string $chunkId,
  681. $data,
  682. $size = null
  683. ): ?array {
  684. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  685. throw new GenericFileException('Object store does not support multipart upload');
  686. }
  687. $cacheEntry = $this->getCache()->get($targetPath);
  688. $urn = $this->getURN($cacheEntry->getId());
  689. $result = $this->objectStore->uploadMultipartPart($urn, $writeToken, (int)$chunkId, $data, $size);
  690. $parts[$chunkId] = [
  691. 'PartNumber' => $chunkId,
  692. 'ETag' => trim($result->get('ETag'), '"'),
  693. ];
  694. return $parts[$chunkId];
  695. }
  696. public function completeChunkedWrite(string $targetPath, string $writeToken): int {
  697. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  698. throw new GenericFileException('Object store does not support multipart upload');
  699. }
  700. $cacheEntry = $this->getCache()->get($targetPath);
  701. $urn = $this->getURN($cacheEntry->getId());
  702. $parts = $this->objectStore->getMultipartUploads($urn, $writeToken);
  703. $sortedParts = array_values($parts);
  704. sort($sortedParts);
  705. try {
  706. $size = $this->objectStore->completeMultipartUpload($urn, $writeToken, $sortedParts);
  707. $stat = $this->stat($targetPath);
  708. $mtime = time();
  709. if (is_array($stat)) {
  710. $stat['size'] = $size;
  711. $stat['mtime'] = $mtime;
  712. $stat['mimetype'] = $this->getMimeType($targetPath);
  713. $this->getCache()->update($stat['fileid'], $stat);
  714. }
  715. } catch (S3MultipartUploadException|S3Exception $e) {
  716. $this->objectStore->abortMultipartUpload($urn, $writeToken);
  717. $this->logger->logException($e, [
  718. 'app' => 'objectstore',
  719. 'message' => 'Could not compete multipart upload ' . $urn . ' with uploadId ' . $writeToken,
  720. ]);
  721. throw new GenericFileException('Could not write chunked file');
  722. }
  723. return $size;
  724. }
  725. public function cancelChunkedWrite(string $targetPath, string $writeToken): void {
  726. if (!$this->objectStore instanceof IObjectStoreMultiPartUpload) {
  727. throw new GenericFileException('Object store does not support multipart upload');
  728. }
  729. $cacheEntry = $this->getCache()->get($targetPath);
  730. $urn = $this->getURN($cacheEntry->getId());
  731. $this->objectStore->abortMultipartUpload($urn, $writeToken);
  732. }
  733. public function setPreserveCacheOnDelete(bool $preserve) {
  734. $this->preserveCacheItemsOnDelete = $preserve;
  735. }
  736. }