FilesMetadataManager.php 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348
  1. <?php
  2. declare(strict_types=1);
  3. /**
  4. * @copyright 2023 Maxence Lange <maxence@artificial-owl.com>
  5. *
  6. * @author Maxence Lange <maxence@artificial-owl.com>
  7. *
  8. * @license GNU AGPL version 3 or any later version
  9. *
  10. * This program is free software: you can redistribute it and/or modify
  11. * it under the terms of the GNU Affero General Public License as
  12. * published by the Free Software Foundation, either version 3 of the
  13. * License, or (at your option) any later version.
  14. *
  15. * This program is distributed in the hope that it will be useful,
  16. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. * GNU Affero General Public License for more details.
  19. *
  20. * You should have received a copy of the GNU Affero General Public License
  21. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  22. *
  23. */
  24. namespace OC\FilesMetadata;
  25. use JsonException;
  26. use OC\FilesMetadata\Job\UpdateSingleMetadata;
  27. use OC\FilesMetadata\Listener\MetadataDelete;
  28. use OC\FilesMetadata\Listener\MetadataUpdate;
  29. use OC\FilesMetadata\Model\FilesMetadata;
  30. use OC\FilesMetadata\Service\IndexRequestService;
  31. use OC\FilesMetadata\Service\MetadataRequestService;
  32. use OCP\BackgroundJob\IJobList;
  33. use OCP\DB\Exception;
  34. use OCP\DB\Exception as DBException;
  35. use OCP\DB\QueryBuilder\IQueryBuilder;
  36. use OCP\EventDispatcher\IEventDispatcher;
  37. use OCP\Files\Cache\CacheEntryRemovedEvent;
  38. use OCP\Files\Events\Node\NodeWrittenEvent;
  39. use OCP\Files\InvalidPathException;
  40. use OCP\Files\Node;
  41. use OCP\Files\NotFoundException;
  42. use OCP\FilesMetadata\Event\MetadataBackgroundEvent;
  43. use OCP\FilesMetadata\Event\MetadataLiveEvent;
  44. use OCP\FilesMetadata\Event\MetadataNamedEvent;
  45. use OCP\FilesMetadata\Exceptions\FilesMetadataException;
  46. use OCP\FilesMetadata\Exceptions\FilesMetadataNotFoundException;
  47. use OCP\FilesMetadata\IFilesMetadataManager;
  48. use OCP\FilesMetadata\IMetadataQuery;
  49. use OCP\FilesMetadata\Model\IFilesMetadata;
  50. use OCP\FilesMetadata\Model\IMetadataValueWrapper;
  51. use OCP\IConfig;
  52. use OCP\IDBConnection;
  53. use Psr\Log\LoggerInterface;
  54. /**
  55. * @inheritDoc
  56. * @since 28.0.0
  57. */
  58. class FilesMetadataManager implements IFilesMetadataManager {
  59. public const CONFIG_KEY = 'files_metadata';
  60. public const MIGRATION_DONE = 'files_metadata_installed';
  61. private const JSON_MAXSIZE = 100000;
  62. private ?IFilesMetadata $all = null;
  63. public function __construct(
  64. private IEventDispatcher $eventDispatcher,
  65. private IJobList $jobList,
  66. private IConfig $config,
  67. private LoggerInterface $logger,
  68. private MetadataRequestService $metadataRequestService,
  69. private IndexRequestService $indexRequestService,
  70. ) {
  71. }
  72. /**
  73. * @inheritDoc
  74. *
  75. * @param Node $node related node
  76. * @param int $process type of process
  77. *
  78. * @return IFilesMetadata
  79. * @throws FilesMetadataException if metadata are invalid
  80. * @throws InvalidPathException if path to file is not valid
  81. * @throws NotFoundException if file cannot be found
  82. * @see self::PROCESS_BACKGROUND
  83. * @see self::PROCESS_LIVE
  84. * @since 28.0.0
  85. */
  86. public function refreshMetadata(
  87. Node $node,
  88. int $process = self::PROCESS_LIVE,
  89. string $namedEvent = ''
  90. ): IFilesMetadata {
  91. try {
  92. $metadata = $this->metadataRequestService->getMetadataFromFileId($node->getId());
  93. } catch (FilesMetadataNotFoundException) {
  94. $metadata = new FilesMetadata($node->getId());
  95. }
  96. // if $process is LIVE, we enforce LIVE
  97. // if $process is NAMED, we go NAMED
  98. // else BACKGROUND
  99. if ((self::PROCESS_LIVE & $process) !== 0) {
  100. $event = new MetadataLiveEvent($node, $metadata);
  101. } elseif ((self::PROCESS_NAMED & $process) !== 0) {
  102. $event = new MetadataNamedEvent($node, $metadata, $namedEvent);
  103. } else {
  104. $event = new MetadataBackgroundEvent($node, $metadata);
  105. }
  106. $this->eventDispatcher->dispatchTyped($event);
  107. $this->saveMetadata($event->getMetadata());
  108. // if requested, we add a new job for next cron to refresh metadata out of main thread
  109. // if $process was set to LIVE+BACKGROUND, we run background process directly
  110. if ($event instanceof MetadataLiveEvent && $event->isRunAsBackgroundJobRequested()) {
  111. if ((self::PROCESS_BACKGROUND & $process) !== 0) {
  112. return $this->refreshMetadata($node, self::PROCESS_BACKGROUND);
  113. }
  114. $this->jobList->add(UpdateSingleMetadata::class, [$node->getOwner()->getUID(), $node->getId()]);
  115. }
  116. return $metadata;
  117. }
  118. /**
  119. * @param int $fileId file id
  120. * @param boolean $generate Generate if metadata does not exists
  121. *
  122. * @inheritDoc
  123. * @return IFilesMetadata
  124. * @throws FilesMetadataNotFoundException if not found
  125. * @since 28.0.0
  126. */
  127. public function getMetadata(int $fileId, bool $generate = false): IFilesMetadata {
  128. try {
  129. return $this->metadataRequestService->getMetadataFromFileId($fileId);
  130. } catch (FilesMetadataNotFoundException $ex) {
  131. if ($generate) {
  132. return new FilesMetadata($fileId);
  133. }
  134. throw $ex;
  135. }
  136. }
  137. /**
  138. * returns metadata of multiple file ids
  139. *
  140. * @param int[] $fileIds file ids
  141. *
  142. * @return array File ID is the array key, files without metadata are not returned in the array
  143. * @psalm-return array<int, IFilesMetadata>
  144. * @since 28.0.0
  145. */
  146. public function getMetadataForFiles(array $fileIds): array {
  147. return $this->metadataRequestService->getMetadataFromFileIds($fileIds);
  148. }
  149. /**
  150. * @param IFilesMetadata $filesMetadata metadata
  151. *
  152. * @inheritDoc
  153. * @throws FilesMetadataException if metadata seems malformed
  154. * @since 28.0.0
  155. */
  156. public function saveMetadata(IFilesMetadata $filesMetadata): void {
  157. if ($filesMetadata->getFileId() === 0 || !$filesMetadata->updated()) {
  158. return;
  159. }
  160. $json = json_encode($filesMetadata->jsonSerialize());
  161. if (strlen($json) > self::JSON_MAXSIZE) {
  162. $this->logger->debug('huge metadata content detected: ' . $json);
  163. throw new FilesMetadataException('json cannot exceed ' . self::JSON_MAXSIZE . ' characters long; fileId: ' . $filesMetadata->getFileId() . '; size: ' . strlen($json));
  164. }
  165. try {
  166. if ($filesMetadata->getSyncToken() === '') {
  167. $this->metadataRequestService->store($filesMetadata);
  168. } else {
  169. $this->metadataRequestService->updateMetadata($filesMetadata);
  170. }
  171. } catch (DBException $e) {
  172. // most of the logged exception are the result of race condition
  173. // between 2 simultaneous process trying to create/update metadata
  174. $this->logger->warning('issue while saveMetadata', ['exception' => $e, 'metadata' => $filesMetadata]);
  175. return;
  176. }
  177. // update indexes
  178. foreach ($filesMetadata->getIndexes() as $index) {
  179. try {
  180. $this->indexRequestService->updateIndex($filesMetadata, $index);
  181. } catch (DBException $e) {
  182. $this->logger->warning('issue while updateIndex', ['exception' => $e]);
  183. }
  184. }
  185. // update metadata types list
  186. $current = $this->getKnownMetadata();
  187. $current->import($filesMetadata->jsonSerialize(true));
  188. $this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current));
  189. }
  190. /**
  191. * @param int $fileId file id
  192. *
  193. * @inheritDoc
  194. * @since 28.0.0
  195. */
  196. public function deleteMetadata(int $fileId): void {
  197. try {
  198. $this->metadataRequestService->dropMetadata($fileId);
  199. } catch (Exception $e) {
  200. $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]);
  201. }
  202. try {
  203. $this->indexRequestService->dropIndex($fileId);
  204. } catch (Exception $e) {
  205. $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileId' => $fileId]);
  206. }
  207. }
  208. /**
  209. * @param IQueryBuilder $qb
  210. * @param string $fileTableAlias alias of the table that contains data about files
  211. * @param string $fileIdField alias of the field that contains file ids
  212. *
  213. * @inheritDoc
  214. * @return IMetadataQuery|null
  215. * @see IMetadataQuery
  216. * @since 28.0.0
  217. */
  218. public function getMetadataQuery(
  219. IQueryBuilder $qb,
  220. string $fileTableAlias,
  221. string $fileIdField
  222. ): ?IMetadataQuery {
  223. if (!$this->metadataInitiated()) {
  224. return null;
  225. }
  226. return new MetadataQuery($qb, $this->getKnownMetadata(), $fileTableAlias, $fileIdField);
  227. }
  228. /**
  229. * @inheritDoc
  230. * @return IFilesMetadata
  231. * @since 28.0.0
  232. */
  233. public function getKnownMetadata(): IFilesMetadata {
  234. if (null !== $this->all) {
  235. return $this->all;
  236. }
  237. $this->all = new FilesMetadata();
  238. try {
  239. $data = json_decode($this->config->getAppValue('core', self::CONFIG_KEY, '[]'), true, 127, JSON_THROW_ON_ERROR);
  240. $this->all->import($data);
  241. } catch (JsonException) {
  242. $this->logger->warning('issue while reading stored list of metadata. Advised to run ./occ files:scan --all --generate-metadata');
  243. }
  244. return $this->all;
  245. }
  246. /**
  247. * @param string $key metadata key
  248. * @param string $type metadata type
  249. * @param bool $indexed TRUE if metadata can be search
  250. * @param int $editPermission remote edit permission via Webdav PROPPATCH
  251. *
  252. * @inheritDoc
  253. * @since 28.0.0
  254. * @see IMetadataValueWrapper::TYPE_INT
  255. * @see IMetadataValueWrapper::TYPE_FLOAT
  256. * @see IMetadataValueWrapper::TYPE_BOOL
  257. * @see IMetadataValueWrapper::TYPE_ARRAY
  258. * @see IMetadataValueWrapper::TYPE_STRING_LIST
  259. * @see IMetadataValueWrapper::TYPE_INT_LIST
  260. * @see IMetadataValueWrapper::TYPE_STRING
  261. * @see IMetadataValueWrapper::EDIT_FORBIDDEN
  262. * @see IMetadataValueWrapper::EDIT_REQ_OWNERSHIP
  263. * @see IMetadataValueWrapper::EDIT_REQ_WRITE_PERMISSION
  264. * @see IMetadataValueWrapper::EDIT_REQ_READ_PERMISSION
  265. */
  266. public function initMetadata(
  267. string $key,
  268. string $type,
  269. bool $indexed = false,
  270. int $editPermission = IMetadataValueWrapper::EDIT_FORBIDDEN
  271. ): void {
  272. $current = $this->getKnownMetadata();
  273. try {
  274. if ($current->getType($key) === $type
  275. && $indexed === $current->isIndex($key)
  276. && $editPermission === $current->getEditPermission($key)) {
  277. return; // if key exists, with same type and indexed, we do nothing.
  278. }
  279. } catch (FilesMetadataNotFoundException) {
  280. // if value does not exist, we keep on the writing of course
  281. }
  282. $current->import([$key => ['type' => $type, 'indexed' => $indexed, 'editPermission' => $editPermission]]);
  283. $this->config->setAppValue('core', self::CONFIG_KEY, json_encode($current));
  284. $this->all = $current;
  285. }
  286. /**
  287. * load listeners
  288. *
  289. * @param IEventDispatcher $eventDispatcher
  290. */
  291. public static function loadListeners(IEventDispatcher $eventDispatcher): void {
  292. $eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class);
  293. $eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class);
  294. }
  295. /**
  296. * Will confirm that tables were created and store an app value to cache the result.
  297. * Can be removed in 29 as this is to avoid strange situation when Nextcloud files were
  298. * replaced but the upgrade was not triggered yet.
  299. *
  300. * @return bool
  301. */
  302. private function metadataInitiated(): bool {
  303. if ($this->config->getAppValue('core', self::MIGRATION_DONE, '0') === '1') {
  304. return true;
  305. }
  306. $dbConnection = \OCP\Server::get(IDBConnection::class);
  307. if ($dbConnection->tableExists(MetadataRequestService::TABLE_METADATA)) {
  308. $this->config->setAppValue('core', self::MIGRATION_DONE, '1');
  309. return true;
  310. }
  311. return false;
  312. }
  313. }