diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000..1ca452e --- /dev/null +++ b/.cursorrules @@ -0,0 +1,29 @@ +# Cursor Rules for S3Proxy Helm Chart Project + +## Project Structure +This is a Helm chart project for deploying S3Proxy to Kubernetes. The chart is located in `charts/s3proxy/`. + +## Important File Handling Rules + +### README.md Files - DO NOT EDIT DIRECTLY +**CRITICAL:** The following README.md files are auto-generated and must NEVER be edited directly: +- `/README.md` (root project README) +- `/charts/s3proxy/README.md` (chart README, if it exists) + +These files are generated from their corresponding template files using helm-docs. + +**When asked to modify README content, you MUST:** +1. ONLY edit the template file: `charts/s3proxy/README.md.gotmpl` +2. NEVER directly edit any `README.md` files +3. Inform the user that README.md is auto-generated and changes should be made to the template +4. After editing the template, remind the user to regenerate README.md using helm-docs + +**Template file location:** +- Template: `charts/s3proxy/README.md.gotmpl` +- Generated: `README.md` (root) - auto-generated from the template + +## Helm Chart Best Practices +- Always validate YAML syntax when editing chart files +- Preserve the existing indentation style (spaces for YAML files as per YAML spec) +- When modifying values.yaml, ensure documentation comments are kept in sync +- Test template rendering with `helm template` after making changes diff --git a/charts/s3proxy/Chart.yaml b/charts/s3proxy/Chart.yaml index 07661c1..e343e43 100644 --- a/charts/s3proxy/Chart.yaml +++ b/charts/s3proxy/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.0.3 +version: 0.0.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/s3proxy/README.md.gotmpl b/charts/s3proxy/README.md.gotmpl index e613139..ed1c067 100644 --- a/charts/s3proxy/README.md.gotmpl +++ b/charts/s3proxy/README.md.gotmpl @@ -53,11 +53,14 @@ The following section lists the configurable parameters of the {{ template "char ```yaml # values-filesystem.yaml config: - authorization: "aws-v4" - identity: "myaccesskey" - credential: "mysecretkey" - backend: - provider: "filesystem-nio2" + auth: + type: "aws-v4" + identity: "myaccesskey" + secret: "mysecretkey" + backends: + filesystem: + enabled: true + nio2: true filesystem: basedir: "/data/s3proxy" @@ -76,9 +79,10 @@ helm install s3proxy-fs ./{{ template "chart.name" . }} -f values-filesystem.yam ```yaml # values-aws-s3.yaml config: - authorization: "aws-v4" - identity: "proxy-access-key" # For clients connecting to s3proxy - credential: "proxy-secret-key" + auth: + type: "aws-v4" + identity: "proxy-access-key" # For clients connecting to s3proxy + secret: "proxy-secret-key" backend: provider: "aws-s3" awsS3: @@ -100,9 +104,10 @@ helm install s3proxy-s3 ./{{ template "chart.name" . }} -f values-aws-s3.yaml ```yaml # values-azure.yaml config: - authorization: "aws-v4" - identity: "myaccesskey" - credential: "mysecretkey" + auth: + type: "aws-v4" + identity: "myaccesskey" + secret: "mysecretkey" backend: provider: "azureblob" azureblob: @@ -123,9 +128,10 @@ helm install s3proxy-azure ./{{ template "chart.name" . }} -f values-azure.yaml ```yaml # values-gcs.yaml config: - authorization: "aws-v4" - identity: "myaccesskey" - credential: "mysecretkey" + auth: + type: "aws-v4" + identity: "myaccesskey" + secret: "mysecretkey" backend: provider: "google-cloud-storage" googleCloudStorage: @@ -145,9 +151,12 @@ persistence: ```yaml # values-anonymous.yaml config: - authorization: "none" - backend: - provider: "transient-nio2" # In-memory storage + auth: + type: "none" + backends: + transient: + enabled: true + nio2: true # In-memory storage persistence: enabled: false @@ -263,7 +272,7 @@ This will remove all resources created by the chart. If using persistence, the P ### Common Issues -1. **Authentication failures**: Ensure `config.identity` and `config.credential` are set correctly for client authentication. +1. **Authentication failures**: Ensure `config.auth.identity` and `config.auth.secret` are set correctly for client authentication. 2. **Backend connection issues**: Verify backend credentials are correctly configured in the appropriate section (e.g., `config.backend.awsS3.*`). diff --git a/charts/s3proxy/override-values.example.yaml b/charts/s3proxy/override-values.example.yaml index c0bb495..1ab76fa 100644 --- a/charts/s3proxy/override-values.example.yaml +++ b/charts/s3proxy/override-values.example.yaml @@ -1,16 +1,31 @@ # Example values for s3proxy Helm chart # This demonstrates a typical configuration using filesystem backend with authentication +# Custom image configuration (optional) +# image: +# repository: andrewgaul/s3proxy +# tag: "latest" +# pullPolicy: Always + +# Custom config merge container image configuration (optional) +# configMergeImage: +# repository: busybox +# tag: "1.36" +# pullPolicy: IfNotPresent + # S3Proxy configuration config: - # Authentication type for clients connecting to S3Proxy - # Options: none, aws-v2, aws-v4, aws-v2-or-v4 - authorization: "aws-v4" - - # Credentials for clients to authenticate to S3Proxy - # These should be changed to secure values - identity: "admin" - credential: "changeme123" + # Log level for S3Proxy (DEBUG, INFO, WARN, ERROR) + logLevel: "INFO" + + auth: + # Authentication type for clients connecting to S3Proxy + # Options: none, aws-v2, aws-v4, aws-v2-or-v4 + type: "aws-v4" + # Credentials for clients to authenticate to S3Proxy + # These should be changed to secure values + identity: "admin" + secret: "changeme123" # Enable CORS for web applications cors: @@ -30,24 +45,95 @@ config: allowCredential: true # Storage backend configuration - backend: - # Use filesystem backend for local storage - provider: "filesystem-nio2" + # Multiple backends can be enabled simultaneously + # Properties files will be loaded in order: main properties first, then each backend's properties + # Later properties can override earlier ones if there are conflicts + backends: + # Filesystem backend for local storage filesystem: + enabled: true # Set to true to use filesystem backend + nio2: true # Set to true for NIO2 implementation (filesystem-nio2), false for standard (filesystem) basedir: "/data/s3proxy" - # Example: AWS S3 backend (uncomment to use) - # provider: "aws-s3" - # awsS3: - # region: "us-west-2" - # accessKeyId: "AKIAIOSFODNN7EXAMPLE" - # secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - - # Example: Azure Blob backend (uncomment to use) - # provider: "azureblob" - # azureblob: - # account: "mystorageaccount" - # key: "base64encodedkey==" + # Transient (in-memory) backend - useful for testing + transient: + enabled: false # Set to true to use transient backend + nio2: true # Set to true for NIO2 implementation (transient-nio2), false for standard (transient) + + # S3 backend (AWS S3 or S3-compatible storage) + s3: + enabled: false # Set to true to use S3 backend + aws: true # Set to true for AWS S3 (aws-s3 provider), false for generic S3 + region: "us-west-2" + # endpoint: "https://s3.amazonaws.com" # Optional custom endpoint (e.g., MinIO, Ceph) + accessKeyId: "AKIAIOSFODNN7EXAMPLE" + secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + + # Azure Blob Storage backend + azureblob: + enabled: false # Set to true to use Azure Blob backend + provider: "azureblob" # Can be "azureblob" or "azureblob-sdk" + account: "mystorageaccount" + key: "base64encodedkey==" + # endpoint: "https://mystorageaccount.blob.core.windows.net" # Optional + # sasToken: "" # Optional SAS token + + # Google Cloud Storage backend + googleCloudStorage: + enabled: false # Set to true to use GCS backend + projectId: "my-project" + + # Service account email or user email (required for both authentication methods) + clientEmail: "service-account@my-project.iam.gserviceaccount.com" + + # Option 1: Using privateKey directly + privateKey: "-----BEGIN RSA PRIVATE KEY-----\n..." + + # Option 2: Using JSON credentials file (preferred for GCP) + # This provides the credential (privateKey) via a mounted file + jsonCredentials: + enabled: false # Set to true to use JSON credentials + + # Either provide the JSON content directly: + # jsonContent: | + # { + # "type": "service_account", + # "project_id": "my-project", + # "private_key_id": "key-id", + # "private_key": "-----BEGIN RSA PRIVATE KEY-----\n...", + # "client_email": "service-account@my-project.iam.gserviceaccount.com", + # "client_id": "...", + # "auth_uri": "https://accounts.google.com/o/oauth2/auth", + # "token_uri": "https://oauth2.googleapis.com/token", + # "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + # "client_x509_cert_url": "..." + # } + + # Or reference an existing secret containing the JSON: + # existingSecret: "my-gcp-credentials-secret" + # secretKey: "credentials.json" # Key in the secret containing the JSON (default: credentials.json) + + # Backblaze B2 backend + b2: + enabled: false # Set to true to use B2 backend + account: "account-id" + applicationKey: "application-key" + + # OpenStack Swift backend + openstackSwift: + enabled: false # Set to true to use Swift backend + authUrl: "https://auth.cloud.com/v2.0" + tenantName: "my-tenant" + userName: "my-user" + password: "my-password" + region: "RegionOne" + + # Rackspace Cloud Files backend + rackspaceCloudfiles: + enabled: false # Set to true to use Rackspace Cloud Files backend + region: "us" # Region: "us" or "uk" + userName: "my-user" + apiKey: "my-api-key" # Persistence settings for filesystem backend persistence: diff --git a/charts/s3proxy/templates/NOTES.txt b/charts/s3proxy/templates/NOTES.txt index cd606ab..f25d4bc 100644 --- a/charts/s3proxy/templates/NOTES.txt +++ b/charts/s3proxy/templates/NOTES.txt @@ -22,8 +22,41 @@ {{- end }} 2. S3Proxy Configuration: - - Backend Provider: {{ .Values.config.backend.provider }} - - Authorization: {{ .Values.config.authorization }} + - Backend Provider: +{{- if .Values.config.backends.filesystem.enabled }} + {{- if .Values.config.backends.filesystem.nio2 }} + filesystem (filesystem-nio2) + {{- else }} + filesystem (filesystem) + {{- end }} +{{- else if .Values.config.backends.transient.enabled }} + {{- if .Values.config.backends.transient.nio2 }} + transient (transient-nio2) + {{- else }} + transient (transient) + {{- end }} +{{- else if .Values.config.backends.s3.enabled }} + {{- if .Values.config.backends.s3.aws }} + AWS S3 (aws-s3) + {{- else }} + Generic S3 (s3) + {{- end }} +{{- else if .Values.config.backends.azureblob.enabled }} + Azure Blob ({{ .Values.config.backends.azureblob.provider }}) +{{- else if .Values.config.backends.googleCloudStorage.enabled }} + Google Cloud Storage +{{- else if .Values.config.backends.b2.enabled }} + Backblaze B2 +{{- else if .Values.config.backends.openstackSwift.enabled }} + OpenStack Swift +{{- else if .Values.config.backends.rackspaceCloudfiles.enabled }} + {{- if eq .Values.config.backends.rackspaceCloudfiles.region "uk" }} + Rackspace Cloud Files UK + {{- else }} + Rackspace Cloud Files US + {{- end }} +{{- end }} + - Authorization: {{ .Values.config.auth.type }} {{- if .Values.config.cors.enabled }} - CORS: Enabled {{- end }} @@ -34,13 +67,13 @@ 3. Testing S3Proxy: Once the service is available, you can test it using AWS CLI or curl: -{{- if eq .Values.config.authorization "none" }} +{{- if eq .Values.config.auth.type "none" }} # Anonymous access (no authentication required) curl -X GET http:/// {{- else }} # Set up AWS CLI credentials (if authentication is enabled) export AWS_ACCESS_KEY_ID= - export AWS_SECRET_ACCESS_KEY= + export AWS_SECRET_ACCESS_KEY= # List buckets aws --endpoint-url http:// s3 ls @@ -53,7 +86,7 @@ # You can also use other AWS S3 API compatible clients like MinIO's mc. - mc alias set s3proxy http:// + mc alias set s3proxy http:// # List buckets mc ls s3proxy @@ -70,12 +103,22 @@ kubectl logs -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "s3proxy.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" 5. Important Notes: -{{- if or (eq .Values.config.backend.provider "filesystem") (eq .Values.config.backend.provider "filesystem-nio2") }} - - Using filesystem backend at: {{ .Values.config.backend.filesystem.basedir }} +{{- if .Values.config.backends.filesystem.enabled }} + - Using filesystem backend at: {{ .Values.config.backends.filesystem.basedir }} {{- end }} -{{- if not .Values.config.identity }} - - WARNING: S3Proxy identity not configured. Remember to set s3proxy.identity and s3proxy.credential for authentication. +{{- if not .Values.config.auth.identity }} + - WARNING: S3Proxy identity not configured. Remember to set config.auth.identity and config.auth.secret for authentication. {{- end }} -{{- if and (ne .Values.config.backend.provider "filesystem") (ne .Values.config.backend.provider "filesystem-nio2") (ne .Values.config.backend.provider "transient") (ne .Values.config.backend.provider "transient-nio2") }} - - Make sure to configure backend credentials for {{ .Values.config.backend.provider }} +{{- if .Values.config.backends.s3.enabled }} + - Make sure to configure S3 backend credentials +{{- else if .Values.config.backends.azureblob.enabled }} + - Make sure to configure Azure Blob backend credentials +{{- else if .Values.config.backends.googleCloudStorage.enabled }} + - Make sure to configure Google Cloud Storage backend credentials +{{- else if .Values.config.backends.b2.enabled }} + - Make sure to configure Backblaze B2 backend credentials +{{- else if .Values.config.backends.openstackSwift.enabled }} + - Make sure to configure OpenStack Swift backend credentials +{{- else if .Values.config.backends.rackspaceCloudfiles.enabled }} + - Make sure to configure Rackspace Cloud Files backend credentials {{- end }} diff --git a/charts/s3proxy/templates/configmap.yaml b/charts/s3proxy/templates/configmap.yaml index 5cfc99c..cc07dc7 100644 --- a/charts/s3proxy/templates/configmap.yaml +++ b/charts/s3proxy/templates/configmap.yaml @@ -1,22 +1,14 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "s3proxy.fullname" . }} - labels: - {{- include "s3proxy.labels" . | nindent 4 }} -data: - s3proxy.properties: | +{{- define "s3proxy.main.config" -}} # S3Proxy configuration s3proxy.endpoint=http://0.0.0.0:{{ .Values.service.targetPort }} - s3proxy.authorization={{ .Values.config.authorization }} + s3proxy.authorization={{ .Values.config.auth.type }} {{- if .Values.config.virtualHost }} s3proxy.virtual-host={{ .Values.config.virtualHost }} {{- end }} -{{- if ne .Values.config.authorization "none" }} - # These will be overridden by environment variables from the secret - s3proxy.identity=${S3PROXY_IDENTITY} - s3proxy.credential=${S3PROXY_CREDENTIAL} +{{- if ne .Values.config.auth.type "none" }} + # Authentication credentials will be merged from the secret properties file + # s3proxy.identity and s3proxy.credential will be provided by the secret {{- end }} {{- if .Values.config.cors.enabled }} @@ -68,57 +60,122 @@ data: # Large object mocking middleware s3proxy.large-object-mocking=true {{- end }} +{{- end }} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "s3proxy.fullname" . }} + labels: + {{- include "s3proxy.labels" . | nindent 4 }} +data: +{{- if .Values.config.backends.filesystem.enabled }} + backend-filesystem.properties: | +{{- include "s3proxy.main.config" . | nindent 4 }} + + # Filesystem backend configuration + {{- if .Values.config.backends.filesystem.nio2 }} + jclouds.provider=filesystem-nio2 + {{- else }} + jclouds.provider=filesystem + {{- end }} + jclouds.filesystem.basedir={{ .Values.config.backends.filesystem.basedir }} +{{- end }} - # JClouds backend configuration - jclouds.provider={{ .Values.config.backend.provider }} +{{- if .Values.config.backends.transient.enabled }} + backend-transient.properties: | +{{- include "s3proxy.main.config" . | nindent 4 }} -{{- if or (eq .Values.config.backend.provider "filesystem") (eq .Values.config.backend.provider "filesystem-nio2") }} - # Filesystem backend - jclouds.filesystem.basedir={{ .Values.config.backend.filesystem.basedir }} -{{- else if or (eq .Values.config.backend.provider "aws-s3") (eq .Values.config.backend.provider "s3") }} - # AWS S3 backend - {{- if .Values.config.backend.awsS3.region }} - jclouds.region={{ .Values.config.backend.awsS3.region }} + # Transient backend configuration + {{- if .Values.config.backends.transient.nio2 }} + jclouds.provider=transient-nio2 + {{- else }} + jclouds.provider=transient {{- end }} - {{- if .Values.config.backend.awsS3.endpoint }} - jclouds.endpoint={{ .Values.config.backend.awsS3.endpoint }} +{{- end }} + +{{- if .Values.config.backends.s3.enabled }} + backend-s3.properties: | +{{- include "s3proxy.main.config" . | nindent 4 }} + + # S3 backend configuration + {{- if .Values.config.backends.s3.aws }} + jclouds.provider=aws-s3 + {{- else }} + jclouds.provider=s3 {{- end }} - # Credentials will be set via environment variables - jclouds.identity=${JCLOUDS_IDENTITY} - jclouds.credential=${JCLOUDS_CREDENTIAL} - {{- else if or (eq .Values.config.backend.provider "azureblob") (eq .Values.config.backend.provider "azureblob-sdk") }} - # Azure Blob backend - {{- if .Values.config.backend.azureblob.endpoint }} - jclouds.azureblob.endpoint={{ .Values.config.backend.azureblob.endpoint }} + {{- if .Values.config.backends.s3.region }} + jclouds.region={{ .Values.config.backends.s3.region }} {{- end }} - # Credentials will be set via environment variables - jclouds.identity=${JCLOUDS_IDENTITY} - jclouds.credential=${JCLOUDS_CREDENTIAL} - {{- if .Values.config.backend.azureblob.sasToken }} - jclouds.azureblob.sas=${JCLOUDS_AZURE_SAS} + {{- if .Values.config.backends.s3.endpoint }} + jclouds.endpoint={{ .Values.config.backends.s3.endpoint }} {{- end }} -{{- else if eq .Values.config.backend.provider "google-cloud-storage" }} - # Google Cloud Storage backend - {{- if .Values.config.backend.googleCloudStorage.projectId }} - jclouds.project-id={{ .Values.config.backend.googleCloudStorage.projectId }} + # Credentials will be merged from the secret properties file + # jclouds.identity and jclouds.credential will be provided by the secret +{{- end }} + +{{- if .Values.config.backends.azureblob.enabled }} + backend-azureblob.properties: | +{{- include "s3proxy.main.config" . | nindent 4 }} + + # Azure Blob backend configuration + jclouds.provider={{ .Values.config.backends.azureblob.provider }} + {{- if .Values.config.backends.azureblob.endpoint }} + jclouds.azureblob.endpoint={{ .Values.config.backends.azureblob.endpoint }} {{- end }} - # Credentials will be set via environment variables - jclouds.identity=${JCLOUDS_IDENTITY} - jclouds.credential=${JCLOUDS_CREDENTIAL} -{{- else if eq .Values.config.backend.provider "b2" }} - # Backblaze B2 backend - # Credentials will be set via environment variables - jclouds.identity=${JCLOUDS_IDENTITY} - jclouds.credential=${JCLOUDS_CREDENTIAL} -{{- else if eq .Values.config.backend.provider "openstack-swift" }} - # OpenStack Swift backend - {{- if .Values.config.backend.swift.authUrl }} - jclouds.keystone.auth-url={{ .Values.config.backend.swift.authUrl }} + # Credentials will be merged from the secret properties file + # jclouds.identity, jclouds.credential, and jclouds.azureblob.sas will be provided by the secret +{{- end }} + +{{- if .Values.config.backends.googleCloudStorage.enabled }} + backend-google-cloud-storage.properties: | +{{- include "s3proxy.main.config" . | nindent 4 }} + + # Google Cloud Storage backend configuration + jclouds.provider=google-cloud-storage + {{- if .Values.config.backends.googleCloudStorage.projectId }} + jclouds.project-id={{ .Values.config.backends.googleCloudStorage.projectId }} {{- end }} - {{- if .Values.config.backend.swift.region }} - jclouds.region={{ .Values.config.backend.swift.region }} + # Credentials will be merged from the secret properties file + # jclouds.identity and jclouds.credential will be provided by the secret +{{- end }} + +{{- if .Values.config.backends.b2.enabled }} + backend-b2.properties: | +{{- include "s3proxy.main.config" . | nindent 4 }} + + # Backblaze B2 backend configuration + jclouds.provider=b2 + # Credentials will be merged from the secret properties file + # jclouds.identity and jclouds.credential will be provided by the secret +{{- end }} + +{{- if .Values.config.backends.openstackSwift.enabled }} + backend-openstack-swift.properties: | +{{- include "s3proxy.main.config" . | nindent 4 }} + + # OpenStack Swift backend configuration + jclouds.provider=openstack-swift + {{- if .Values.config.backends.openstackSwift.authUrl }} + jclouds.keystone.auth-url={{ .Values.config.backends.openstackSwift.authUrl }} {{- end }} - # Credentials will be set via environment variables - jclouds.identity=${JCLOUDS_IDENTITY} - jclouds.credential=${JCLOUDS_CREDENTIAL} + {{- if .Values.config.backends.openstackSwift.region }} + jclouds.region={{ .Values.config.backends.openstackSwift.region }} + {{- end }} + # Credentials will be merged from the secret properties file + # jclouds.identity and jclouds.credential will be provided by the secret {{- end }} + +{{- if .Values.config.backends.rackspaceCloudfiles.enabled }} + backend-rackspace-cloudfiles.properties: | +{{- include "s3proxy.main.config" . | nindent 4 }} + + # Rackspace Cloud Files backend configuration + {{- if eq .Values.config.backends.rackspaceCloudfiles.region "uk" }} + jclouds.provider=rackspace-cloudfiles-uk + {{- else }} + jclouds.provider=rackspace-cloudfiles-us + {{- end }} + # Credentials will be merged from the secret properties file + # jclouds.identity and jclouds.credential will be provided by the secret +{{- end }} \ No newline at end of file diff --git a/charts/s3proxy/templates/deployment.yaml b/charts/s3proxy/templates/deployment.yaml index 25eb7fe..df62c26 100644 --- a/charts/s3proxy/templates/deployment.yaml +++ b/charts/s3proxy/templates/deployment.yaml @@ -29,6 +29,54 @@ spec: serviceAccountName: {{ include "s3proxy.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: merge-configs + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.configMergeImage.repository }}:{{ .Values.configMergeImage.tag }}" + imagePullPolicy: {{ .Values.configMergeImage.pullPolicy }} + command: ["sh", "-c"] + args: + - | + set -e + echo "Merging configuration files..." + + # Check if secret properties file exists + SECRET_FILE="/secret/secret.properties" + + # Loop through all properties files in the config directory + for config_file in /config/*.properties + do + if [ -f "$config_file" ] + then + # Get the filename + filename=$(basename "$config_file") + output_file="/merged-config/$filename" + + echo "Processing $filename..." + + # Copy base config file to output + cp "$config_file" "$output_file" + + # If secret file exists, append its contents (overriding duplicates) + if [ -f "$SECRET_FILE" ] + then + echo "" >> "$output_file" # Add newline separator + cat "$SECRET_FILE" >> "$output_file" + fi + fi + done + + echo "Configuration files merged successfully." + volumeMounts: + - name: config + mountPath: /config + readOnly: true + - name: secret-config + mountPath: /secret + readOnly: true + - name: merged-config + mountPath: /merged-config containers: - name: {{ .Chart.Name }} securityContext: @@ -36,14 +84,43 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: - - /opt/s3proxy/run-docker-container.sh + - java args: + - -DLOG_LEVEL={{ .Values.config.logLevel | upper }} + - -jar + - /opt/s3proxy/s3proxy +{{- if .Values.config.backends.filesystem.enabled }} - "--properties" - - "/config/s3proxy.properties" - envFrom: - # Inject all environment variables from the secret - - secretRef: - name: {{ include "s3proxy.fullname" . }} + - "/merged-config/backend-filesystem.properties" +{{- end }} +{{- if .Values.config.backends.transient.enabled }} + - "--properties" + - "/merged-config/backend-transient.properties" +{{- end }} +{{- if .Values.config.backends.s3.enabled }} + - "--properties" + - "/merged-config/backend-s3.properties" +{{- end }} +{{- if .Values.config.backends.azureblob.enabled }} + - "--properties" + - "/merged-config/backend-azureblob.properties" +{{- end }} +{{- if .Values.config.backends.googleCloudStorage.enabled }} + - "--properties" + - "/merged-config/backend-google-cloud-storage.properties" +{{- end }} +{{- if .Values.config.backends.b2.enabled }} + - "--properties" + - "/merged-config/backend-b2.properties" +{{- end }} +{{- if .Values.config.backends.openstackSwift.enabled }} + - "--properties" + - "/merged-config/backend-openstack-swift.properties" +{{- end }} +{{- if .Values.config.backends.rackspaceCloudfiles.enabled }} + - "--properties" + - "/merged-config/backend-rackspace-cloudfiles.properties" +{{- end }} {{- with .Values.extraEnvVars }} env: {{- toYaml . | nindent 12 }} @@ -53,8 +130,7 @@ spec: containerPort: {{ .Values.service.targetPort }} protocol: TCP livenessProbe: - httpGet: - path: / + tcpSocket: port: http initialDelaySeconds: 30 periodSeconds: 10 @@ -62,8 +138,7 @@ spec: successThreshold: 1 failureThreshold: 3 readinessProbe: - httpGet: - path: / + tcpSocket: port: http initialDelaySeconds: 10 periodSeconds: 5 @@ -73,15 +148,20 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - - name: config - mountPath: /config + - name: merged-config + mountPath: /merged-config readOnly: true -{{- if or (eq .Values.config.backend.provider "filesystem") (eq .Values.config.backend.provider "filesystem-nio2") }} +{{- if .Values.config.backends.filesystem.enabled }} {{- if .Values.persistence.enabled }} - name: data - mountPath: {{ .Values.config.backend.filesystem.basedir }} + mountPath: {{ .Values.config.backends.filesystem.basedir }} {{- end }} {{- end }} +{{- if and .Values.config.backends.googleCloudStorage.enabled .Values.config.backends.googleCloudStorage.jsonCredentials.enabled }} + - name: gcp-json-credentials + mountPath: /gcp-credentials + readOnly: true +{{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -89,13 +169,27 @@ spec: - name: config configMap: name: {{ include "s3proxy.fullname" . }} -{{- if or (eq .Values.config.backend.provider "filesystem") (eq .Values.config.backend.provider "filesystem-nio2") }} + - name: secret-config + secret: + secretName: {{ include "s3proxy.fullname" . }} + - name: merged-config + emptyDir: {} +{{- if .Values.config.backends.filesystem.enabled }} {{- if .Values.persistence.enabled }} - name: data persistentVolumeClaim: claimName: {{ .Values.persistence.existingClaim | default (include "s3proxy.fullname" .) }} {{- end }} {{- end }} +{{- if and .Values.config.backends.googleCloudStorage.enabled .Values.config.backends.googleCloudStorage.jsonCredentials.enabled }} + - name: gcp-json-credentials + secret: + {{- if .Values.config.backends.googleCloudStorage.jsonCredentials.jsonContent }} + secretName: {{ include "s3proxy.fullname" . }}-gcp-json + {{- else if .Values.config.backends.googleCloudStorage.jsonCredentials.existingSecret }} + secretName: {{ .Values.config.backends.googleCloudStorage.jsonCredentials.existingSecret }} + {{- end }} +{{- end }} {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/s3proxy/templates/pvc.yaml b/charts/s3proxy/templates/pvc.yaml index 0d18193..b56d585 100644 --- a/charts/s3proxy/templates/pvc.yaml +++ b/charts/s3proxy/templates/pvc.yaml @@ -1,4 +1,4 @@ -{{- if and (or (eq .Values.config.backend.provider "filesystem") (eq .Values.config.backend.provider "filesystem-nio2")) .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} +{{- if and .Values.config.backends.filesystem.enabled .Values.persistence.enabled (not .Values.persistence.existingClaim) -}} apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/charts/s3proxy/templates/secret-gcp-json.yaml b/charts/s3proxy/templates/secret-gcp-json.yaml new file mode 100644 index 0000000..522a4d8 --- /dev/null +++ b/charts/s3proxy/templates/secret-gcp-json.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.config.backends.googleCloudStorage.enabled .Values.config.backends.googleCloudStorage.jsonCredentials.enabled .Values.config.backends.googleCloudStorage.jsonCredentials.jsonContent }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "s3proxy.fullname" . }}-gcp-json + labels: + {{- include "s3proxy.labels" . | nindent 4 }} +type: Opaque +stringData: + {{ .Values.config.backends.googleCloudStorage.jsonCredentials.secretKey }}: | + {{- .Values.config.backends.googleCloudStorage.jsonCredentials.jsonContent | nindent 4 }} +{{- end }} diff --git a/charts/s3proxy/templates/secret.yaml b/charts/s3proxy/templates/secret.yaml index e6dfd10..0c6371f 100644 --- a/charts/s3proxy/templates/secret.yaml +++ b/charts/s3proxy/templates/secret.yaml @@ -5,58 +5,73 @@ metadata: labels: {{- include "s3proxy.labels" . | nindent 4 }} type: Opaque -data: -{{- if and .Values.config.identity .Values.config.credential }} - # S3Proxy authentication credentials (for clients connecting to s3proxy) - # Using uppercase names to match environment variable conventions - S3PROXY_IDENTITY: {{ .Values.config.identity | b64enc | quote }} - S3PROXY_CREDENTIAL: {{ .Values.config.credential | b64enc | quote }} +stringData: + # Properties file containing sensitive configuration values + # This will be merged with the backend properties files by the init container + secret.properties: | +{{- if and .Values.config.auth.identity .Values.config.auth.secret }} + # S3Proxy authentication credentials (for clients connecting to s3proxy) + s3proxy.identity={{ .Values.config.auth.identity }} + s3proxy.credential={{ .Values.config.auth.secret }} {{- end }} - -{{- if or (eq .Values.config.backend.provider "aws-s3") (eq .Values.config.backend.provider "s3") }} - # AWS S3 backend credentials - {{- if .Values.config.backend.awsS3.accessKeyId }} - JCLOUDS_IDENTITY: {{ .Values.config.backend.awsS3.accessKeyId | b64enc | quote }} - {{- end }} - {{- if .Values.config.backend.awsS3.secretAccessKey }} - JCLOUDS_CREDENTIAL: {{ .Values.config.backend.awsS3.secretAccessKey | b64enc | quote }} - {{- end }} -{{- else if or (eq .Values.config.backend.provider "azureblob") (eq .Values.config.backend.provider "azureblob-sdk") }} - # Azure Blob backend credentials - {{- if .Values.config.backend.azureblob.account }} - JCLOUDS_IDENTITY: {{ .Values.config.backend.azureblob.account | b64enc | quote }} - {{- end }} - {{- if .Values.config.backend.azureblob.key }} - JCLOUDS_CREDENTIAL: {{ .Values.config.backend.azureblob.key | b64enc | quote }} - {{- end }} - {{- if .Values.config.backend.azureblob.sasToken }} - JCLOUDS_AZURE_SAS: {{ .Values.config.backend.azureblob.sasToken | b64enc | quote }} - {{- end }} - {{- else if eq .Values.config.backend.provider "google-cloud-storage" }} - # Google Cloud Storage backend credentials - {{- if .Values.config.backend.googleCloudStorage.clientEmail }} - JCLOUDS_IDENTITY: {{ .Values.config.backend.googleCloudStorage.clientEmail | b64enc | quote }} - {{- end }} - {{- if .Values.config.backend.googleCloudStorage.privateKey }} - JCLOUDS_CREDENTIAL: {{ .Values.config.backend.googleCloudStorage.privateKey | b64enc | quote }} - {{- end }} - {{- else if eq .Values.config.backend.provider "b2" }} - # Backblaze B2 backend credentials - {{- if .Values.config.backend.b2.account }} - JCLOUDS_IDENTITY: {{ .Values.config.backend.b2.account | b64enc | quote }} - {{- end }} - {{- if .Values.config.backend.b2.applicationKey }} - JCLOUDS_CREDENTIAL: {{ .Values.config.backend.b2.applicationKey | b64enc | quote }} + +{{- if .Values.config.backends.s3.enabled }} + # S3 backend credentials + {{- if .Values.config.backends.s3.accessKeyId }} + jclouds.identity={{ .Values.config.backends.s3.accessKeyId }} + {{- end }} + {{- if .Values.config.backends.s3.secretAccessKey }} + jclouds.credential={{ .Values.config.backends.s3.secretAccessKey }} + {{- end }} +{{- else if .Values.config.backends.azureblob.enabled }} + # Azure Blob backend credentials + {{- if .Values.config.backends.azureblob.account }} + jclouds.identity={{ .Values.config.backends.azureblob.account }} + {{- end }} + {{- if .Values.config.backends.azureblob.key }} + jclouds.credential={{ .Values.config.backends.azureblob.key }} + {{- end }} + {{- if .Values.config.backends.azureblob.sasToken }} + jclouds.azureblob.sas={{ .Values.config.backends.azureblob.sasToken }} + {{- end }} +{{- else if .Values.config.backends.googleCloudStorage.enabled }} + # Google Cloud Storage backend credentials + {{- if .Values.config.backends.googleCloudStorage.clientEmail }} + jclouds.identity={{ .Values.config.backends.googleCloudStorage.clientEmail }} + {{- end }} + {{- if .Values.config.backends.googleCloudStorage.jsonCredentials.enabled }} + # Using JSON credentials file + jclouds.credential=/gcp-credentials/{{ .Values.config.backends.googleCloudStorage.jsonCredentials.secretKey }} + {{- else }} + # Using privateKey directly + {{- if .Values.config.backends.googleCloudStorage.privateKey }} + jclouds.credential={{ .Values.config.backends.googleCloudStorage.privateKey }} {{- end }} - {{- else if eq .Values.config.backend.provider "openstack-swift" }} - # OpenStack Swift backend credentials - {{- if .Values.config.backend.swift.tenantName }} - {{- if .Values.config.backend.swift.userName }} - JCLOUDS_IDENTITY: {{ printf "%s:%s" .Values.config.backend.swift.tenantName .Values.config.backend.swift.userName | b64enc | quote }} - {{- end }} + {{- end }} +{{- else if .Values.config.backends.b2.enabled }} + # Backblaze B2 backend credentials + {{- if .Values.config.backends.b2.account }} + jclouds.identity={{ .Values.config.backends.b2.account }} + {{- end }} + {{- if .Values.config.backends.b2.applicationKey }} + jclouds.credential={{ .Values.config.backends.b2.applicationKey }} + {{- end }} +{{- else if .Values.config.backends.openstackSwift.enabled }} + # OpenStack Swift backend credentials + {{- if .Values.config.backends.openstackSwift.tenantName }} + {{- if .Values.config.backends.openstackSwift.userName }} + jclouds.identity={{ printf "%s:%s" .Values.config.backends.openstackSwift.tenantName .Values.config.backends.openstackSwift.userName }} {{- end }} - {{- if .Values.config.backend.swift.password }} - JCLOUDS_CREDENTIAL: {{ .Values.config.backend.swift.password | b64enc | quote }} {{- end }} -{{- end }} - + {{- if .Values.config.backends.openstackSwift.password }} + jclouds.credential={{ .Values.config.backends.openstackSwift.password }} + {{- end }} +{{- else if .Values.config.backends.rackspaceCloudfiles.enabled }} + # Rackspace Cloud Files backend credentials + {{- if .Values.config.backends.rackspaceCloudfiles.userName }} + jclouds.identity={{ .Values.config.backends.rackspaceCloudfiles.userName }} + {{- end }} + {{- if .Values.config.backends.rackspaceCloudfiles.apiKey }} + jclouds.credential={{ .Values.config.backends.rackspaceCloudfiles.apiKey }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/s3proxy/values.yaml b/charts/s3proxy/values.yaml index 65724a3..3ad4155 100644 --- a/charts/s3proxy/values.yaml +++ b/charts/s3proxy/values.yaml @@ -13,6 +13,14 @@ image: # -- Overrides the image tag whose default is the chart appVersion tag: "" +configMergeImage: + # -- Config merge container image repository + repository: busybox + # -- Config merge container image pull policy + pullPolicy: IfNotPresent + # -- Config merge container image tag + tag: "1.36" + # -- Image pull secrets imagePullSecrets: [] # -- String to partially override s3proxy.fullname template @@ -41,9 +49,9 @@ service: # -- Kubernetes service type type: ClusterIP # -- Service port - port: 8080 + port: 9000 # -- Target port (controls both the container port and S3Proxy bind port) - targetPort: 8080 + targetPort: 9000 # -- Service annotations annotations: {} @@ -102,13 +110,16 @@ tolerations: [] affinity: {} config: - # -- Authorization type (none, aws-v2, aws-v4, aws-v2-or-v4) - authorization: "aws-v4" + # -- Log level for S3Proxy (DEBUG, INFO, WARN, ERROR) + logLevel: "INFO" - # -- S3 Access Key ID for client authentication - identity: "" - # -- S3 Secret Access Key for client authentication - credential: "" + auth: + # -- Authorization type (none, aws-v2, aws-v4, aws-v2-or-v4) + type: "aws-v4" + # -- S3 Access Key ID for client authentication + identity: "" + # -- S3 Secret Access Key for client authentication + secret: "" # -- Virtual Host configuration virtualHost: "" @@ -146,25 +157,40 @@ config: # -- Enable large object mocking largeObjectMocking: false - backend: - # -- Backend provider type (filesystem, filesystem-nio2, transient, transient-nio2, aws-s3, s3, azureblob, azureblob-sdk, b2, google-cloud-storage, openstack-swift, rackspace-cloudfiles-uk, rackspace-cloudfiles-us) - provider: "filesystem-nio2" - + backends: filesystem: + # -- Enable filesystem backend + enabled: true + # -- Use NIO2 implementation (filesystem-nio2) instead of standard filesystem + nio2: true # -- Base directory for filesystem backend basedir: "/data/s3proxy" - awsS3: + transient: + # -- Enable transient (in-memory) backend + enabled: false + # -- Use NIO2 implementation (transient-nio2) instead of standard transient + nio2: true + + s3: + # -- Enable S3 backend + enabled: false + # -- Use AWS-specific S3 provider (aws-s3) instead of generic S3 provider + aws: true # -- AWS region region: "" # -- S3 endpoint endpoint: "" - # -- AWS Access Key ID for backend + # -- S3 Access Key ID for backend accessKeyId: "" - # -- AWS Secret Access Key for backend + # -- S3 Secret Access Key for backend secretAccessKey: "" azureblob: + # -- Enable Azure Blob Storage backend + enabled: false + # -- Provider type (azureblob or azureblob-sdk) + provider: "azureblob" # -- Storage account name account: "" # -- Storage account key @@ -175,24 +201,36 @@ config: sasToken: "" googleCloudStorage: + # -- Enable Google Cloud Storage backend + enabled: false # -- GCP project ID projectId: "" - # -- Private key ID - privateKeyId: "" - # -- Private key + # -- Private key (only used when jsonCredentials.enabled is false) privateKey: "" - # -- Service account email + # -- Service account email or user email (used with both privateKey and jsonCredentials methods) clientEmail: "" - # -- Client ID - clientId: "" + # -- JSON credentials configuration + jsonCredentials: + # -- Use JSON credentials file instead of privateKey + enabled: false + # -- JSON content for creating a new secret (takes precedence over existingSecret) + jsonContent: "" + # -- Name of existing secret containing GCP credentials JSON + existingSecret: "" + # -- Key in the secret containing the JSON credentials (default: credentials.json) + secretKey: "credentials.json" b2: + # -- Enable Backblaze B2 backend + enabled: false # -- B2 account ID account: "" # -- B2 application key applicationKey: "" - swift: + openstackSwift: + # -- Enable OpenStack Swift backend + enabled: false # -- Authentication URL authUrl: "" # -- Tenant name @@ -204,6 +242,16 @@ config: # -- Region region: "" + rackspaceCloudfiles: + # -- Enable Rackspace Cloud Files backend + enabled: false + # -- Region (uk or us) + region: "us" + # -- Username + userName: "" + # -- API key + apiKey: "" + persistence: # -- Enable persistence using PVC enabled: true