1
0

S3ConnectionTrait.php 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. <?php
  2. /**
  3. * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
  4. *
  5. * @author Arthur Schiwon <blizzz@arthur-schiwon.de>
  6. * @author Christoph Wurst <christoph@winzerhof-wurst.at>
  7. * @author Florent <florent@coppint.com>
  8. * @author James Letendre <James.Letendre@gmail.com>
  9. * @author Morris Jobke <hey@morrisjobke.de>
  10. * @author Robin Appelman <robin@icewind.nl>
  11. * @author Roeland Jago Douma <roeland@famdouma.nl>
  12. * @author S. Cat <33800996+sparrowjack63@users.noreply.github.com>
  13. * @author Stephen Cuppett <steve@cuppett.com>
  14. * @author Jasper Weyne <jasperweyne@gmail.com>
  15. *
  16. * @license GNU AGPL version 3 or any later version
  17. *
  18. * This program is free software: you can redistribute it and/or modify
  19. * it under the terms of the GNU Affero General Public License as
  20. * published by the Free Software Foundation, either version 3 of the
  21. * License, or (at your option) any later version.
  22. *
  23. * This program is distributed in the hope that it will be useful,
  24. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  25. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  26. * GNU Affero General Public License for more details.
  27. *
  28. * You should have received a copy of the GNU Affero General Public License
  29. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  30. *
  31. */
  32. namespace OC\Files\ObjectStore;
  33. use Aws\ClientResolver;
  34. use Aws\Credentials\CredentialProvider;
  35. use Aws\Credentials\Credentials;
  36. use Aws\Exception\CredentialsException;
  37. use Aws\S3\Exception\S3Exception;
  38. use Aws\S3\S3Client;
  39. use GuzzleHttp\Promise;
  40. use GuzzleHttp\Promise\RejectedPromise;
  41. use OCP\ICertificateManager;
  42. use Psr\Log\LoggerInterface;
  43. trait S3ConnectionTrait {
  44. use S3ConfigTrait;
  45. protected string $id;
  46. protected bool $test;
  47. protected ?S3Client $connection = null;
  48. protected function parseParams($params) {
  49. if (empty($params['bucket'])) {
  50. throw new \Exception("Bucket has to be configured.");
  51. }
  52. $this->id = 'amazon::' . $params['bucket'];
  53. $this->test = isset($params['test']);
  54. $this->bucket = $params['bucket'];
  55. // Default to 5 like the S3 SDK does
  56. $this->concurrency = $params['concurrency'] ?? 5;
  57. $this->proxy = $params['proxy'] ?? false;
  58. $this->timeout = $params['timeout'] ?? 15;
  59. $this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD';
  60. $this->uploadPartSize = $params['uploadPartSize'] ?? 524288000;
  61. $this->putSizeLimit = $params['putSizeLimit'] ?? 104857600;
  62. $this->copySizeLimit = $params['copySizeLimit'] ?? 5242880000;
  63. $this->useMultipartCopy = (bool)($params['useMultipartCopy'] ?? true);
  64. $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
  65. $params['s3-accelerate'] = $params['hostname'] == 's3-accelerate.amazonaws.com' || $params['hostname'] == 's3-accelerate.dualstack.amazonaws.com';
  66. $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname'];
  67. if (!isset($params['port']) || $params['port'] === '') {
  68. $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
  69. }
  70. $params['verify_bucket_exists'] = $params['verify_bucket_exists'] ?? true;
  71. if ($params['s3-accelerate']) {
  72. $params['verify_bucket_exists'] = false;
  73. }
  74. $this->params = $params;
  75. }
  76. public function getBucket() {
  77. return $this->bucket;
  78. }
  79. public function getProxy() {
  80. return $this->proxy;
  81. }
  82. /**
  83. * Returns the connection
  84. *
  85. * @return S3Client connected client
  86. * @throws \Exception if connection could not be made
  87. */
  88. public function getConnection() {
  89. if ($this->connection !== null) {
  90. return $this->connection;
  91. }
  92. $scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https';
  93. $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';
  94. // Adding explicit credential provider to the beginning chain.
  95. // Including default credential provider (skipping AWS shared config files).
  96. $provider = CredentialProvider::memoize(
  97. CredentialProvider::chain(
  98. $this->paramCredentialProvider(),
  99. CredentialProvider::defaultProvider(['use_aws_shared_config_files' => false])
  100. )
  101. );
  102. $options = [
  103. 'version' => $this->params['version'] ?? 'latest',
  104. 'credentials' => $provider,
  105. 'endpoint' => $base_url,
  106. 'region' => $this->params['region'],
  107. 'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false,
  108. 'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()),
  109. 'csm' => false,
  110. 'use_arn_region' => false,
  111. 'http' => ['verify' => $this->getCertificateBundlePath()],
  112. 'use_aws_shared_config_files' => false,
  113. ];
  114. if ($this->params['s3-accelerate']) {
  115. $options['use_accelerate_endpoint'] = true;
  116. } else {
  117. $options['endpoint'] = $base_url;
  118. }
  119. if ($this->getProxy()) {
  120. $options['http']['proxy'] = $this->getProxy();
  121. }
  122. if (isset($this->params['legacy_auth']) && $this->params['legacy_auth']) {
  123. $options['signature_version'] = 'v2';
  124. }
  125. $this->connection = new S3Client($options);
  126. if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
  127. $logger = \OC::$server->get(LoggerInterface::class);
  128. $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.',
  129. ['app' => 'objectstore']);
  130. }
  131. if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) {
  132. $logger = \OC::$server->get(LoggerInterface::class);
  133. try {
  134. $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']);
  135. if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
  136. throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket);
  137. }
  138. $this->connection->createBucket(['Bucket' => $this->bucket]);
  139. $this->testTimeout();
  140. } catch (S3Exception $e) {
  141. $logger->debug('Invalid remote storage.', [
  142. 'exception' => $e,
  143. 'app' => 'objectstore',
  144. ]);
  145. if ($e->getAwsErrorCode() !== "BucketAlreadyOwnedByYou") {
  146. throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage());
  147. }
  148. }
  149. }
  150. // google cloud's s3 compatibility doesn't like the EncodingType parameter
  151. if (strpos($base_url, 'storage.googleapis.com')) {
  152. $this->connection->getHandlerList()->remove('s3.auto_encode');
  153. }
  154. return $this->connection;
  155. }
  156. /**
  157. * when running the tests wait to let the buckets catch up
  158. */
  159. private function testTimeout() {
  160. if ($this->test) {
  161. sleep($this->timeout);
  162. }
  163. }
  164. public static function legacySignatureProvider($version, $service, $region) {
  165. switch ($version) {
  166. case 'v2':
  167. case 's3':
  168. return new S3Signature();
  169. default:
  170. return null;
  171. }
  172. }
  173. /**
  174. * This function creates a credential provider based on user parameter file
  175. */
  176. protected function paramCredentialProvider(): callable {
  177. return function () {
  178. $key = empty($this->params['key']) ? null : $this->params['key'];
  179. $secret = empty($this->params['secret']) ? null : $this->params['secret'];
  180. if ($key && $secret) {
  181. return Promise\promise_for(
  182. new Credentials($key, $secret)
  183. );
  184. }
  185. $msg = 'Could not find parameters set for credentials in config file.';
  186. return new RejectedPromise(new CredentialsException($msg));
  187. };
  188. }
  189. protected function getCertificateBundlePath(): ?string {
  190. if ((int)($this->params['use_nextcloud_bundle'] ?? "0")) {
  191. // since we store the certificate bundles on the primary storage, we can't get the bundle while setting up the primary storage
  192. if (!isset($this->params['primary_storage'])) {
  193. /** @var ICertificateManager $certManager */
  194. $certManager = \OC::$server->get(ICertificateManager::class);
  195. return $certManager->getAbsoluteBundlePath();
  196. } else {
  197. return \OC::$SERVERROOT . '/resources/config/ca-bundle.crt';
  198. }
  199. } else {
  200. return null;
  201. }
  202. }
  203. protected function getSSECKey(): ?string {
  204. if (isset($this->params['sse_c_key'])) {
  205. return $this->params['sse_c_key'];
  206. }
  207. return null;
  208. }
  209. protected function getSSECParameters(bool $copy = false): array {
  210. $key = $this->getSSECKey();
  211. if ($key === null) {
  212. return [];
  213. }
  214. $rawKey = base64_decode($key);
  215. if ($copy) {
  216. return [
  217. 'CopySourceSSECustomerAlgorithm' => 'AES256',
  218. 'CopySourceSSECustomerKey' => $rawKey,
  219. 'CopySourceSSECustomerKeyMD5' => md5($rawKey, true)
  220. ];
  221. }
  222. return [
  223. 'SSECustomerAlgorithm' => 'AES256',
  224. 'SSECustomerKey' => $rawKey,
  225. 'SSECustomerKeyMD5' => md5($rawKey, true)
  226. ];
  227. }
  228. }