Merge pull request #44252 from nextcloud/enh/s3-connectivity

feat(ObjectStore): Make the S3 `concurrency` option configurable

Author: susnux

lib/composer/composer/autoload_classmap.php

@@ -1419,6 +1419,7 @@
     'OC\\Files\\ObjectStore\\ObjectStoreScanner' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
     'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',
     'OC\\Files\\ObjectStore\\S3' => $baseDir . '/lib/private/Files/ObjectStore/S3.php',
+    'OC\\Files\\ObjectStore\\S3ConfigTrait' => $baseDir . '/lib/private/Files/ObjectStore/S3ConfigTrait.php',
     'OC\\Files\\ObjectStore\\S3ConnectionTrait' => $baseDir . '/lib/private/Files/ObjectStore/S3ConnectionTrait.php',
     'OC\\Files\\ObjectStore\\S3ObjectTrait' => $baseDir . '/lib/private/Files/ObjectStore/S3ObjectTrait.php',
     'OC\\Files\\ObjectStore\\S3Signature' => $baseDir . '/lib/private/Files/ObjectStore/S3Signature.php',

lib/composer/composer/autoload_static.php

@@ -1452,6 +1452,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Files\\ObjectStore\\ObjectStoreScanner' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
         'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',
         'OC\\Files\\ObjectStore\\S3' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3.php',
+        'OC\\Files\\ObjectStore\\S3ConfigTrait' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3ConfigTrait.php',
         'OC\\Files\\ObjectStore\\S3ConnectionTrait' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3ConnectionTrait.php',
         'OC\\Files\\ObjectStore\\S3ObjectTrait' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3ObjectTrait.php',
         'OC\\Files\\ObjectStore\\S3Signature' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/S3Signature.php',

lib/private/Files/ObjectStore/S3ConfigTrait.php

@@ -0,0 +1,53 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2024 Ferdinand Thiessen <opensource@fthiessen.de>
+ *
+ * @author Ferdinand Thiessen <opensource@fthiessen.de>
+ *
+ * @license GNU AGPL-3.0-or-later
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Files\ObjectStore;
+
+/**
+ * Shared configuration between ConnectionTrait and ObjectTrait to ensure both to be in sync
+ */
+trait S3ConfigTrait {
+	protected array $params;
+
+	protected string $bucket;
+
+	/** Maximum number of concurrent multipart uploads */
+	protected int $concurrency;
+
+	protected int $timeout;
+
+	protected string $proxy;
+
+	protected string $storageClass;
+
+	protected int $uploadPartSize;
+
+	private int $putSizeLimit;
+
+	private int $copySizeLimit;
+
+	private bool $useMultipartCopy = true;
+}

lib/private/Files/ObjectStore/S3ConnectionTrait.php

@@ -44,39 +44,13 @@
 use Psr\Log\LoggerInterface;
 
 trait S3ConnectionTrait {
-	/** @var array */
-	protected $params;
+	use S3ConfigTrait;
 
-	/** @var S3Client */
-	protected $connection;
+	protected string $id;
 
-	/** @var string */
-	protected $id;
+	protected bool $test;
 
-	/** @var string */
-	protected $bucket;
-
-	/** @var int */
-	protected $timeout;
-
-	/** @var string */
-	protected $proxy;
-
-	/** @var string */
-	protected $storageClass;
-
-	/** @var int */
-	protected $uploadPartSize;
-
-	/** @var int */
-	private $putSizeLimit;
-
-	/** @var int */
-	private $copySizeLimit;
-
-	private bool $useMultipartCopy = true;
-
-	protected $test;
+	protected ?S3Client $connection = null;
 
 	protected function parseParams($params) {
 		if (empty($params['bucket'])) {
@@ -87,6 +61,8 @@ protected function parseParams($params) {
 
 		$this->test = isset($params['test']);
 		$this->bucket = $params['bucket'];
+		// Default to 5 like the S3 SDK does
+		$this->concurrency = $params['concurrency'] ?? 5;
 		$this->proxy = $params['proxy'] ?? false;
 		$this->timeout = $params['timeout'] ?? 15;
 		$this->storageClass = !empty($params['storageClass']) ? $params['storageClass'] : 'STANDARD';
@@ -118,7 +94,7 @@ public function getProxy() {
 	 * @throws \Exception if connection could not be made
 	 */
 	public function getConnection() {
-		if (!is_null($this->connection)) {
+		if ($this->connection !== null) {
 			return $this->connection;
 		}
 

lib/private/Files/ObjectStore/S3ObjectTrait.php

@@ -36,6 +36,8 @@
 use Psr\Http\Message\StreamInterface;
 
 trait S3ObjectTrait {
+	use S3ConfigTrait;
+
 	/**
 	 * Returns the connection
 	 *
@@ -127,6 +129,7 @@ protected function writeSingle(string $urn, StreamInterface $stream, string $mim
 	protected function writeMultiPart(string $urn, StreamInterface $stream, string $mimetype = null): void {
 		$uploader = new MultipartUploader($this->getConnection(), $stream, [
 			'bucket' => $this->bucket,
+			'concurrency' => $this->concurrency,
 			'key' => $urn,
 			'part_size' => $this->uploadPartSize,
 			'params' => [