fix: wip

Author: ServerSideHannes

benchmarks/bench.py

@@ -25,14 +25,12 @@
 
 import aioboto3
 
-# Object sizes
+# Object sizes for realistic single-part uploads
 SIZES = {
     "tiny": 1024,           # 1 KB
     "small": 64 * 1024,     # 64 KB
     "medium": 1024 * 1024,  # 1 MB
     "large": 10 * 1024 * 1024,  # 10 MB
-    "xlarge": 100 * 1024 * 1024,  # 100 MB
-    "huge": 1024 * 1024 * 1024,  # 1 GiB
 }
 
 # Endpoints

benchmarks/docker-compose.yml

@@ -68,9 +68,9 @@ services:
       S3PROXY_ENCRYPT_KEY: benchmark-test-key-32-bytes-!!
       AWS_ACCESS_KEY_ID: benchmarkadminuser
       AWS_SECRET_ACCESS_KEY: benchmarkadminpassword
-      # Higher limits for benchmarking
-      S3PROXY_MAX_CONCURRENT_UPLOADS: "50"
-      S3PROXY_MAX_CONCURRENT_DOWNLOADS: "50"
+      # Throttle for realistic single-part uploads (10MB files)
+      # Memory: 10MB + 64MB = 74MB per upload, 10 concurrent = 740MB < 1GB
+      S3PROXY_THROTTLING_REQUESTS_MAX: "10"
       S3PROXY_NO_TLS: "true"
       S3PROXY_LOG_LEVEL: WARNING  # Reduce log noise during benchmarks
       S3PROXY_REDIS_URL: redis://redis:6379/0
@@ -82,8 +82,8 @@ services:
     # Required for py-spy profiling
     cap_add:
       - SYS_PTRACE
-    mem_limit: 512m
-    memswap_limit: 512m
+    mem_limit: 1g
+    memswap_limit: 1g
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:4433/readyz"]
       interval: 2s

e2e/docker-compose.e2e.yml

@@ -37,35 +37,23 @@ services:
     ports:
       - "8080:4433"
     environment:
-      # S3 backend configuration
       S3PROXY_HOST: http://minio:9000
       S3PROXY_REGION: us-east-1
-
-      # Encryption key (32 bytes for AES-256)
       S3PROXY_ENCRYPT_KEY: test-key-for-e2e-testing-32bytes
-
-      # AWS credentials for MinIO access
-      AWS_ACCESS_KEY_ID: minioadmin
-      AWS_SECRET_ACCESS_KEY: minioadmin
-
-      # Concurrency limits
-      S3PROXY_MAX_CONCURRENT_UPLOADS: "5"
-      S3PROXY_MAX_CONCURRENT_DOWNLOADS: "5"
-
-      # Server configuration
       S3PROXY_NO_TLS: "true"
       S3PROXY_LOG_LEVEL: INFO
-
-      # Redis configuration
       S3PROXY_REDIS_URL: redis://redis:6379/0
+      S3PROXY_THROTTLING_REQUESTS_MAX: "10"
+      S3PROXY_MAX_UPLOAD_SIZE_MB: "45"
+      AWS_ACCESS_KEY_ID: minioadmin
+      AWS_SECRET_ACCESS_KEY: minioadmin
     depends_on:
       minio:
         condition: service_healthy
       redis:
         condition: service_healthy
-    # Hard memory limit (same as Kubernetes)
-    mem_limit: 512m
-    memswap_limit: 512m  # Same as mem_limit = no swap = hard OOM kill
+    mem_limit: 1280m
+    memswap_limit: 1280m
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:4433/readyz"]
       interval: 5s

manifests/templates/configmap.yaml

@@ -14,9 +14,8 @@ data:
   S3PROXY_IP: "0.0.0.0"
   S3PROXY_PORT: {{ .Values.server.port | quote }}
   S3PROXY_NO_TLS: {{ .Values.server.noTls | quote }}
-  S3PROXY_MAX_CONCURRENT_UPLOADS: {{ .Values.performance.maxConcurrentUploads | quote }}
-  S3PROXY_MAX_CONCURRENT_DOWNLOADS: {{ .Values.performance.maxConcurrentDownloads | quote }}
-  S3PROXY_AUTO_MULTIPART_MB: {{ .Values.performance.autoMultipartMb | quote }}
+  S3PROXY_THROTTLING_REQUESTS_MAX: {{ .Values.performance.throttlingRequestsMax | quote }}
+  S3PROXY_MAX_UPLOAD_SIZE_MB: {{ .Values.performance.maxUploadSizeMb | quote }}
   {{- if index .Values "redis-ha" "enabled" }}
   S3PROXY_REDIS_URL: "redis://{{ .Release.Name }}-redis-ha-haproxy:6379/0"
   {{- else }}

manifests/values.yaml

@@ -1,52 +1,32 @@
 # S3Proxy Helm Chart Values
 
-# Deployment settings
 replicaCount: 3
 
-# Container image
 image:
-  # IMPORTANT: Change to your image registry
-  # Example: ghcr.io/myorg/sseproxy-python or your private registry
   repository: ghcr.io/YOUR_USERNAME/sseproxy-python
-  # IMPORTANT: Never use 'latest' in production - use specific version tags
-  # Example: "v0.1.0", "v0.2.0", etc.
   tag: latest
   pullPolicy: IfNotPresent
 
-# S3 configuration (used only when minio.enabled=false)
-# Ignored if MinIO is enabled - MinIO will be used as the S3 backend
+# S3 configuration (used when minio.enabled=false)
 s3:
-  # S3-compatible endpoint: AWS S3, DigitalOcean Spaces, etc.
-  # Examples:
-  #   - "s3.amazonaws.com" (AWS S3)
-  #   - "s3.us-west-2.amazonaws.com" (AWS S3 specific region)
-  #   - "nyc3.digitaloceanspaces.com" (DigitalOcean Spaces)
   host: "s3.amazonaws.com"
-  # AWS region (ignored for non-AWS S3 services)
   region: "us-east-1"
 
-# Server settings
 server:
-  port: 4433  # Listen port (should match service.port)
-  noTls: true  # TLS termination handled by Ingress or Load Balancer
+  port: 4433
+  noTls: true
 
-# Performance tuning settings
 performance:
-  maxConcurrentUploads: 10  # Max concurrent upload operations
-  maxConcurrentDownloads: 10  # Max concurrent download operations
-  autoMultipartMb: 16  # Chunk size in MB for multipart uploads
+  throttlingRequestsMax: 10
+  maxUploadSizeMb: 45
 
-# MinIO configuration (embedded S3 backend)
-# Set enabled: false to use external S3 (AWS, DigitalOcean Spaces, etc.)
-# When disabled, configure s3.host and set secrets.awsAccessKeyId/awsSecretAccessKey
+# Embedded MinIO (set enabled: false to use external S3)
 minio:
-  enabled: true  # For production, consider external S3 service
+  enabled: true
   image:
     repository: minio/minio
-    tag: latest  # Specify version for production, e.g., "RELEASE.2024-01-16T16-07-38Z"
+    tag: latest
     pullPolicy: IfNotPresent
-  # IMPORTANT: Change credentials in production!
-  # Default credentials below are for development only
   rootUser: "minioadmin"
   rootPassword: "minioadmin"
   resources:
@@ -57,38 +37,22 @@ minio:
       cpu: "500m"
       memory: "512Mi"
 
-# Redis cache for upload state management
-# Choose one: redis-ha (for HA) or externalRedis (for managed services)
+# External Redis (for managed services)
 externalRedis:
-  # Use this for managed Redis: AWS ElastiCache, Azure Cache, Redis Cloud, etc.
-  # Leave empty to use redis-ha instead
-  # Format: redis://host:port/db or redis://:password@host:port/db
-  url: ""  # e.g., "redis://my-elasticache.abc123.cache.amazonaws.com:6379/0"
-  # Include password in URL if needed: redis://:mypassword@host:port/db
-  # TTL for upload state in hours
+  url: ""
   uploadTtlHours: 24
 
-# Redis HA configuration (embedded high-availability Redis)
-# Uses dandydev/redis-ha chart with Sentinel for automatic failover
-# Set enabled: false if using externalRedis instead
+# Redis HA (embedded)
 redis-ha:
-  enabled: true  # Disable if using managed Redis service
-  # Number of Redis replicas (1 master + N-1 replicas)
+  enabled: true
   replicas: 3
+  existingSecret: ""
 
-  # Use existing secret for Redis password (RECOMMENDED for production)
-  # If provided, auth and authKey below are ignored
-  # Create with: kubectl create secret generic redis-password --from-literal=redis-password="your-password"
-  existingSecret: ""  # Name of existing secret with key "redis-password"
-
-  # Persistence configuration
   persistentVolume:
     enabled: true
     size: 10Gi
-    storageClass: ""  # Use default storage class, or specify e.g., "gp3", "standard"
+    storageClass: ""
 
-  # HAProxy for single-endpoint access (recommended)
-  # This allows standard redis:// URLs without sentinel-aware client code
   haproxy:
     enabled: true
     replicas: 2
@@ -100,7 +64,6 @@ redis-ha:
         cpu: "200m"
         memory: "128Mi"
 
-  # Sentinel configuration
   sentinel:
     port: 26379
     quorum: 2
@@ -109,22 +72,17 @@ redis-ha:
       failover-timeout: 180000
       parallel-syncs: 5
 
-  # Redis configuration
   redis:
     port: 6379
     config:
       maxmemory-policy: volatile-lru
       min-replicas-to-write: 1
       min-replicas-max-lag: 5
 
-  # Security - Redis password authentication (ignored if existingSecret is set above)
-  auth: false  # Set to true to enable password protection
-  authKey: ""  # Redis password (required if auth=true and no existingSecret, generate with: openssl rand -base64 32)
-
-  # Pod anti-affinity for HA (spread across nodes)
+  auth: false
+  authKey: ""
   hardAntiAffinity: true
 
-  # Resource limits for Redis pods
   resources:
     requests:
       cpu: "100m"
@@ -133,54 +91,30 @@ redis-ha:
       cpu: "500m"
       memory: "512Mi"
 
-# Secret Configuration
-# IMPORTANT: Never commit actual secrets to git!
-# Priority: existingSecrets > create static secret
-
+# Secrets (use existing secret in production)
 secrets:
-  # Option 1: Use existing Secret (RECOMMENDED for production)
-  # Reference an existing Kubernetes secret and optionally map its keys
-  # If using default key names, just set the secret name:
-  #   kubectl create secret generic my-s3-secrets \
-  #     --from-literal=S3PROXY_ENCRYPT_KEY="$(openssl rand -base64 32)" \
-  #     --from-literal=AWS_ACCESS_KEY_ID="AKIA..." \
-  #     --from-literal=AWS_SECRET_ACCESS_KEY="..."
   existingSecrets:
     enabled: false
-    name: ""  # Name of existing Kubernetes secret
-    # Optional: Map secret keys if using different key names
+    name: ""
     keys:
-      encryptKey: "S3PROXY_ENCRYPT_KEY"  # Secret key name for encryption key
-      awsAccessKeyId: "AWS_ACCESS_KEY_ID"  # Secret key name for access key
-      awsSecretAccessKey: "AWS_SECRET_ACCESS_KEY"  # Secret key name for secret key
-
-  # Option 2: Create new secret from values (use only for development)
-  # For production, use existingSecrets with a pre-created secret
-  # Provide values via helm --set or secure values file, never hardcode here
+      encryptKey: "S3PROXY_ENCRYPT_KEY"
+      awsAccessKeyId: "AWS_ACCESS_KEY_ID"
+      awsSecretAccessKey: "AWS_SECRET_ACCESS_KEY"
 
-  # S3PROXY_ENCRYPT_KEY: AES-256-GCM encryption key (base64-encoded 32 bytes)
-  # Generate with: openssl rand -base64 32
   encryptKey: ""
-
-  # AWS/S3 credentials (ignored if minio.enabled=true and using MinIO defaults)
-  # Only needed when: minio.enabled=false AND using external S3
   awsAccessKeyId: ""
   awsSecretAccessKey: ""
 
-# Logging
-logLevel: "INFO"  # Options: DEBUG, INFO, WARNING, ERROR
+logLevel: "INFO"
 
-# Resource limits for s3proxy pods
-# Adjust based on your workload and cluster capacity
 resources:
   requests:
     cpu: "100m"
-    memory: "256Mi"
+    memory: "512Mi"
   limits:
     cpu: "1000m"
-    memory: "512Mi"
+    memory: "1Gi"
 
-# Kubernetes Service configuration
 service:
-  type: ClusterIP  # Use LoadBalancer for external access, or configure Ingress
-  port: 4433  # Service port (container also runs on this port)
+  type: ClusterIP
+  port: 4433

s3proxy/config.py

@@ -27,14 +27,11 @@ class Settings(BaseSettings):
     cert_path: str = Field(default="/etc/s3proxy/certs", description="TLS certificate directory")
 
     # Performance settings
-    throttling_requests_max: int = Field(default=0, description="Max concurrent requests")
-    max_single_encrypted_mb: int = Field(default=16, description="Max single-part object size (MB)")
-    auto_multipart_mb: int = Field(default=16, description="Auto-multipart threshold (MB)")
-    max_concurrent_uploads: int = Field(default=10, description="Max concurrent uploads")
-    max_concurrent_downloads: int = Field(default=10, description="Max concurrent downloads")
-
-    # Feature flags
-    allow_multipart: bool = Field(default=False, description="Allow unencrypted multipart")
+    # Memory usage: file_size + ~64MB per concurrent upload
+    # For 1GB pod with 10MB files: ~13 concurrent safe, default 10 for margin
+    # Files >16MB automatically use multipart encryption
+    throttling_requests_max: int = Field(default=10, description="Max concurrent requests (0=unlimited)")
+    max_upload_size_mb: int = Field(default=45, description="Max single-request upload size (MB)")
 
     # Redis settings (for distributed state)
     redis_url: str = Field(default="redis://localhost:6379/0", description="Redis connection URL")
@@ -55,14 +52,9 @@ def kek(self) -> bytes:
         return hashlib.sha256(self.encrypt_key.encode()).digest()
 
     @property
-    def max_single_encrypted_bytes(self) -> int:
-        """Max single encrypted object size in bytes."""
-        return self.max_single_encrypted_mb * 1024 * 1024
-
-    @property
-    def auto_multipart_bytes(self) -> int:
-        """Auto-multipart threshold in bytes."""
-        return self.auto_multipart_mb * 1024 * 1024
+    def max_upload_size_bytes(self) -> int:
+        """Max upload size in bytes."""
+        return self.max_upload_size_mb * 1024 * 1024
 
     @property
     def s3_endpoint(self) -> str:

s3proxy/handlers/objects.py

@@ -264,11 +264,13 @@ async def handle_put_object(self, request: Request, creds: S3Credentials) -> Res
         if needs_chunked_decode:
             body = decode_aws_chunked(body)
 
-        if self.settings.auto_multipart_bytes > 0 and len(body) > self.settings.auto_multipart_bytes:
-            return await self._put_multipart(client, bucket, key, body, content_type)
+        # Reject if exceeds max upload size
+        if len(body) > self.settings.max_upload_size_bytes:
+            raise HTTPException(413, f"Max upload size: {self.settings.max_upload_size_mb}MB")
 
-        if len(body) > self.settings.max_single_encrypted_bytes:
-            raise HTTPException(413, f"Max size: {self.settings.max_single_encrypted_mb}MB")
+        # Auto-use multipart for files >16MB to split encryption into parts
+        if len(body) > crypto.PART_SIZE:
+            return await self._put_multipart(client, bucket, key, body, content_type)
 
         encrypted = crypto.encrypt_object(body, self.settings.kek)
         etag = hashlib.md5(body).hexdigest()
@@ -385,10 +387,11 @@ async def upload_part(data: bytes) -> None:
                 total_plaintext_size += len(chunk)
 
                 # Upload when buffer reaches PART_SIZE
+                # Process immediately without intermediate variable to reduce memory
                 while len(buffer) >= crypto.PART_SIZE:
-                    part_data = bytes(buffer[:crypto.PART_SIZE])
+                    # Extract, upload, then clear - minimizes peak memory
+                    await upload_part(bytes(buffer[:crypto.PART_SIZE]))
                     del buffer[:crypto.PART_SIZE]
-                    await upload_part(part_data)
 
             # Upload remaining data
             if buffer: