From ae15e3fbd145a52513e485bcaf2ef6db087356aa Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Sat, 18 Jul 2026 06:55:20 +0000 Subject: [PATCH] feat(aws): add AWS Marketplace target to s3-cloud --- .github/workflows/packer-ci-aws.yml | 237 ++++++++++++++++++ .github/workflows/packer-ci.yml | 36 ++- .github/workflows/packer-release-aws.yml | 141 +++++++++++ .github/workflows/packer-release.yml | 17 +- .gitignore | 5 + deploy/aws/.gitignore | 1 + deploy/aws/AGENTS.md | 117 +++++++++ deploy/aws/MARKETPLACE_README.md | 32 +++ deploy/aws/Makefile | 63 +++++ deploy/aws/README.md | 107 ++++++++ deploy/aws/cloudformation.template.json | 213 ++++++++++++++++ deploy/aws/iam-policy.json | 112 +++++++++ deploy/aws/marketplace/README.md | 9 + deploy/aws/marketplace/SECURITY_GROUPS.md | 18 ++ .../aws/marketplace/SUBMISSION_CHECKLIST.md | 32 +++ deploy/aws/marketplace/USAGE_INSTRUCTIONS.md | 39 +++ deploy/aws/scripts/000-aws-credentials.sh | 54 ++++ deploy/aws/scripts/900-cleanup.sh | 58 +++++ deploy/aws/scripts/prune-instances.py | 63 +++++ deploy/aws/scripts/prune-snapshots.py | 69 +++++ deploy/aws/scripts/validate.sh | 166 ++++++++++++ deploy/aws/submit.py | 80 ++++++ deploy/aws/template.pkr.hcl | 211 ++++++++++++++++ .../shared/scripts/lib/compliance-checks.sh | 21 +- docs/platform-support.md | 38 ++- packer/scripts/install-s3-server.sh | 4 +- scripts/validate-aws.sh | 91 +++++++ 27 files changed, 1995 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/packer-ci-aws.yml create mode 100644 .github/workflows/packer-release-aws.yml create mode 100644 deploy/aws/.gitignore create mode 100644 deploy/aws/AGENTS.md create mode 100644 deploy/aws/MARKETPLACE_README.md create mode 100644 deploy/aws/Makefile create mode 100644 deploy/aws/README.md create mode 100644 deploy/aws/cloudformation.template.json create mode 100644 deploy/aws/iam-policy.json create mode 100644 deploy/aws/marketplace/README.md create mode 100644 deploy/aws/marketplace/SECURITY_GROUPS.md create mode 100644 deploy/aws/marketplace/SUBMISSION_CHECKLIST.md create mode 100644 deploy/aws/marketplace/USAGE_INSTRUCTIONS.md create mode 100644 deploy/aws/scripts/000-aws-credentials.sh create mode 100755 deploy/aws/scripts/900-cleanup.sh create mode 100755 deploy/aws/scripts/prune-instances.py create mode 100755 deploy/aws/scripts/prune-snapshots.py create mode 100755 deploy/aws/scripts/validate.sh create mode 100755 deploy/aws/submit.py create mode 100644 deploy/aws/template.pkr.hcl create mode 100755 scripts/validate-aws.sh diff --git a/.github/workflows/packer-ci-aws.yml b/.github/workflows/packer-ci-aws.yml new file mode 100644 index 0000000..f44b06c --- /dev/null +++ b/.github/workflows/packer-ci-aws.yml @@ -0,0 +1,237 @@ +# CI workflow for AWS builds. +# Triggered by packer-ci.yml dispatcher via workflow_dispatch (gh workflow run --ref). +# Builds an AMI, validates it, cleans up on failure. +# AMIs are kept until PR close (dispatcher sends cleanup-mode=close) or +# push-to-develop (ephemeral, cleaned inline). + +name: CI AWS + +on: + workflow_call: + inputs: + snapshot-prefix: + description: 'Snapshot prefix (e.g. pinner-s3-aws-pr-123- or pinner-s3-aws-ci-)' + required: true + type: string + cleanup-mode: + description: 'Cleanup mode: none, inline (delete after validate), or close (delete all PR AMIs)' + required: false + default: 'inline' + type: string + pr-number: + description: 'PR number (for close cleanup)' + required: false + type: string + rescue-password: + description: 'Set root password for serial console rescue access (debug only, never for release)' + required: false + type: string + workflow_dispatch: + inputs: + snapshot-prefix: + description: 'Snapshot prefix (e.g. pinner-s3-aws-pr-123- or pinner-s3-aws-ci-)' + required: true + type: string + cleanup-mode: + description: 'Cleanup mode: none, inline (delete after validate), or close (delete all PR AMIs)' + required: false + default: 'inline' + type: choice + options: + - 'inline' + - 'none' + - 'close' + pr-number: + description: 'PR number (for close cleanup)' + required: false + type: string + rescue-password: + description: 'Set root password for serial console rescue access (debug only, never for release)' + required: false + type: string + +concurrency: + group: ci-aws-${{ github.ref }} + cancel-in-progress: false + +jobs: + build-and-validate: + if: inputs.cleanup-mode != 'close' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + env: + VENDOR: aws + SNAPSHOT_PREFIX: ${{ inputs.snapshot-prefix }} + CLEANUP_MODE: ${{ inputs.cleanup-mode }} + PR_NUMBER: ${{ inputs.pr-number }} + steps: + - name: Checkout repo + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Setup SSH key + env: + CI_BUILD_SSH_KEY: ${{ secrets.CI_BUILD_SSH_KEY }} + run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh + if echo "$CI_BUILD_SSH_KEY" | base64 -d > ~/.ssh/id_ed25519 2>/dev/null; then + : # base64 decoded + else + echo "$CI_BUILD_SSH_KEY" > ~/.ssh/id_ed25519 + fi + chmod 600 ~/.ssh/id_ed25519 + + - name: Setup Packer + uses: hashicorp/setup-packer@v3 + with: + version: latest + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Discover VPC and Subnet + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + run: | + echo "==> Discovering default VPC..." + VPC_ID=$(aws ec2 describe-vpcs --filters "Name=isDefault,Values=true" --query 'Vpcs[0].VpcId' --output text) + if [ "$VPC_ID" = "None" ] || [ -z "$VPC_ID" ]; then + echo "Error: No default VPC found. Please create a default VPC or specify VPC_ID explicitly." + exit 1 + fi + echo "VPC_ID=$VPC_ID" + + echo "==> Discovering public subnet in AZ that supports t3.small..." + # Query subnets and their AZs, then filter out us-east-1e (no t3 support) + SUBNET_INFO=$(aws ec2 describe-subnets \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=map-public-ip-on-launch,Values=true" \ + --query 'Subnets[*].[SubnetId,AvailabilityZone]' --output text) + + SUBNET_ID="" + while read -r sub az; do + if [ "$az" != "us-east-1e" ]; then + SUBNET_ID="$sub" + echo "Found subnet $SUBNET_ID in AZ $az" + break + fi + done <<< "$SUBNET_INFO" + + if [ -z "$SUBNET_ID" ]; then + echo "Error: No suitable public subnet found outside us-east-1e." + exit 1 + fi + + echo "VPC_ID=$VPC_ID" >> "$GITHUB_ENV" + echo "SUBNET_ID=$SUBNET_ID" >> "$GITHUB_ENV" + + - name: Build AMI + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + PKR_VAR_snapshot_prefix: ${{ steps.config.outputs.snapshot-prefix }} + PKR_VAR_vpc_id: ${{ steps.vpc.outputs.vpc_id }} + PKR_VAR_subnet_id: ${{ steps.vpc.outputs.subnet_id }} + run: | + cd deploy/"$VENDOR" + make build SNAPSHOT_PREFIX="$SNAPSHOT_PREFIX" VPC_ID="$VPC_ID" SUBNET_ID="$SUBNET_ID" + + - name: Validate AMI + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + VPC_ID: ${{ env.VPC_ID }} + SUBNET_ID: ${{ env.SUBNET_ID }} + RESCUE_PASSWORD: ${{ github.event.inputs.rescue-password }} + run: | + cd deploy/"$VENDOR" + make validate VPC_ID="$VPC_ID" SUBNET_ID="$SUBNET_ID" + + - name: Upload manifest + if: always() + uses: actions/upload-artifact@v4 + with: + name: manifest-aws + path: deploy/aws/manifest.json + if-no-files-found: ignore + retention-days: 7 + + - name: Cleanup on success (inline mode) + if: success() && inputs.cleanup-mode == 'inline' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd deploy/"$VENDOR" + if [ -f manifest.json ]; then + echo "==> Cleaning up AMI (inline mode)..." + make cleanup-snapshot || echo "Warning: manifest cleanup failed, daily prune will catch it" + fi + + - name: Cleanup on failure + if: failure() && github.event.inputs.skip-cleanup == 'false' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd deploy/"$VENDOR" + if [ -f manifest.json ]; then + echo "==> Cleaning up AMI from failed build/validate..." + make cleanup-snapshot || echo "Warning: manifest cleanup failed, daily prune will catch it" + fi + # Prune any orphaned AMIs with this prefix + python3 scripts/prune-snapshots.py --max-age-hours 0 --prefix "$SNAPSHOT_PREFIX" 2>/dev/null \ + || echo "Warning: prune fallback failed, daily prune will catch it" + + cleanup: + if: inputs.cleanup-mode == 'close' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + env: + VENDOR: aws + PR_NUMBER: ${{ inputs.pr-number }} + steps: + - name: Checkout repo + uses: actions/checkout@v5 + with: + fetch-depth: 1 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Delete PR AMIs + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ -z "$PR_NUMBER" ]; then + echo "No PR number, skipping cleanup." + exit 0 + fi + PREFIX="pinner-s3-aws-pr-${PR_NUMBER}-" + echo "==> Cleaning up AMIs with prefix: $PREFIX" + cd deploy/aws + python3 scripts/prune-snapshots.py --max-age-hours 0 --prefix "$PREFIX" diff --git a/.github/workflows/packer-ci.yml b/.github/workflows/packer-ci.yml index 3fedbf7..b7b4129 100644 --- a/.github/workflows/packer-ci.yml +++ b/.github/workflows/packer-ci.yml @@ -40,7 +40,7 @@ on: workflow_dispatch: inputs: vendor: - description: 'Vendor to build (digitalocean, vultr, or all)' + description: 'Vendor to build (digitalocean, vultr, aws, or all)' required: false default: 'all' type: choice @@ -48,6 +48,7 @@ on: - 'all' - 'digitalocean' - 'vultr' + - 'aws' skip-cleanup: description: 'Skip cleanup after build (keep snapshot for manual testing)' required: false @@ -56,6 +57,10 @@ on: options: - 'true' - 'false' + rescue-password: + description: 'Set root password for serial console rescue access (AWS debug only)' + required: false + type: string concurrency: group: packer-ci-${{ github.ref }} @@ -82,7 +87,7 @@ jobs: run: | # PR close: cleanup all vendors if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && [ "$GITHUB_EVENT_ACTION" = "closed" ]; then - echo 'vendors=["digitalocean","vultr"]' >> "$GITHUB_OUTPUT" + echo 'vendors=["digitalocean","vultr","aws"]' >> "$GITHUB_OUTPUT" echo "is-closed=true" >> "$GITHUB_OUTPUT" echo "Detected vendors (PR close): all" exit 0 @@ -93,7 +98,7 @@ jobs: if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then INPUT="${{ github.event.inputs.vendor }}" if [ "$INPUT" = "all" ]; then - echo 'vendors=["digitalocean","vultr"]' >> "$GITHUB_OUTPUT" + echo 'vendors=["digitalocean","vultr","aws"]' >> "$GITHUB_OUTPUT" else echo "vendors=[\"$INPUT\"]" >> "$GITHUB_OUTPUT" fi @@ -116,6 +121,9 @@ jobs: if echo "$CHANGED" | grep -q '^deploy/vultr/'; then VENDORS="${VENDORS}vultr " fi + if echo "$CHANGED" | grep -q '^deploy/aws/'; then + VENDORS="${VENDORS}aws " + fi # Shared files affect ALL vendors — add them additively. if echo "$CHANGED" | grep -qE '^(packer/scripts/|deploy/shared/|docker-compose\.yml|sidecar/)'; then @@ -125,6 +133,9 @@ jobs: if ! echo "$VENDORS" | grep -q 'vultr'; then VENDORS="${VENDORS}vultr " fi + if ! echo "$VENDORS" | grep -q 'aws'; then + VENDORS="${VENDORS}aws " + fi fi # Build JSON array from space-separated list @@ -210,6 +221,7 @@ jobs: additional_files: >- deploy/${{ matrix.vendor }}/scripts/900-cleanup.sh deploy/${{ matrix.vendor }}/scripts/validate.sh + deploy/${{ matrix.vendor }}/scripts/001_provision.sh - name: Shellcheck (DO application-tag, digitalocean only) if: matrix.vendor == 'digitalocean' @@ -228,6 +240,12 @@ jobs: deploy/shared/files/var/lib/cloud/scripts/per-instance/001_onboot deploy/shared/files/etc/update-motd.d/99-one-click + - name: Shellcheck (repo-level AWS validator) + if: matrix.vendor == 'aws' + uses: ludeeus/action-shellcheck@master + with: + additional_files: 'scripts/validate-aws.sh' + - name: Python compile check run: python3 -m py_compile deploy/${{ matrix.vendor }}/submit.py deploy/${{ matrix.vendor }}/scripts/prune-snapshots.py deploy/${{ matrix.vendor }}/scripts/prune-instances.py @@ -330,3 +348,15 @@ jobs: -f cleanup-mode="$CLEANUP_MODE" \ -f pr-number="$PR_NUMBER" echo "Dispatched CI Vultr on ref $REF" + + dispatch-aws: + needs: [detect-vendors, check-permissions] + if: always() && !cancelled() && needs.check-permissions.outputs.authorized == 'true' && contains(needs.detect-vendors.outputs.vendors, 'aws') && needs.detect-vendors.outputs.is-closed != 'true' + uses: ./.github/workflows/packer-ci-aws.yml + secrets: inherit + with: + snapshot-prefix: pinner-s3-aws-pr-${{ github.event.pull_request.number }}- + cleanup-mode: none + pr-number: ${{ github.event.pull_request.number }} + rescue-password: ${{ github.event.inputs.rescue-password }} + diff --git a/.github/workflows/packer-release-aws.yml b/.github/workflows/packer-release-aws.yml new file mode 100644 index 0000000..50391fd --- /dev/null +++ b/.github/workflows/packer-release-aws.yml @@ -0,0 +1,141 @@ +# Release workflow for AWS marketplace. +# Triggered by packer-release.yml dispatcher via workflow_dispatch (gh workflow run --ref). +# Builds an AMI, validates it, submits to the AWS Marketplace. + +name: Release AWS + +on: + workflow_call: + inputs: + version: + description: 'Release version (e.g. 1.0.0)' + required: true + type: string + reason: + description: 'Release reason (e.g. initial-release)' + required: false + default: 'initial-release' + type: string + workflow_dispatch: + inputs: + version: + description: 'Release version (e.g. 1.0.0)' + required: true + type: string + reason: + description: 'Release reason (e.g. initial-release)' + required: false + default: 'initial-release' + type: string + +jobs: + build-and-validate: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + env: + VENDOR: aws + SNAPSHOT_PREFIX: pinner-s3-aws- + VERSION: ${{ inputs.version }} + steps: + - name: Checkout repo + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Setup SSH key + env: + CI_BUILD_SSH_KEY: ${{ secrets.CI_BUILD_SSH_KEY }} + run: | + mkdir -p ~/.ssh + chmod 700 ~/.ssh + if echo "$CI_BUILD_SSH_KEY" | base64 -d > ~/.ssh/id_ed25519 2>/dev/null; then + : # base64 decoded + else + echo "$CI_BUILD_SSH_KEY" > ~/.ssh/id_ed25519 + fi + chmod 600 ~/.ssh/id_ed25519 + + - name: Setup Packer + uses: hashicorp/setup-packer@v3 + with: + version: latest + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Build AMI + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd deploy/"$VENDOR" + make build SNAPSHOT_PREFIX="$SNAPSHOT_PREFIX" VERSION="$VERSION" + + - name: Validate AMI + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + run: | + cd deploy/"$VENDOR" + make validate + + - name: Upload manifest + if: always() + uses: actions/upload-artifact@v4 + with: + name: manifest-aws-release + path: deploy/aws/manifest.json + if-no-files-found: ignore + retention-days: 30 + + submit: + needs: build-and-validate + if: success() + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v5 + with: + fetch-depth: 1 + + - name: Download manifest + uses: actions/download-artifact@v4 + with: + name: manifest-aws-release + path: deploy/aws/ + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Submit to AWS Marketplace + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-1 + VERSION: ${{ inputs.version }} + REASON: ${{ inputs.reason }} + run: | + cd deploy/aws + if [ -f manifest.json ]; then + echo "==> Submitting AMI to AWS Marketplace..." + python3 submit.py --manifest manifest.json --version "$VERSION" --reason "$REASON" + else + echo "ERROR: manifest.json not found. Cannot submit." + exit 1 + fi diff --git a/.github/workflows/packer-release.yml b/.github/workflows/packer-release.yml index cd76c80..a39484d 100644 --- a/.github/workflows/packer-release.yml +++ b/.github/workflows/packer-release.yml @@ -17,7 +17,7 @@ on: workflow_dispatch: inputs: vendor: - description: 'Target vendor (digitalocean, vultr, or all)' + description: 'Target vendor (digitalocean, vultr, aws, or all)' required: false default: 'all' type: choice @@ -25,6 +25,7 @@ on: - 'all' - 'digitalocean' - 'vultr' + - 'aws' reason: description: 'Reason for update (shown in Vendor Portal / Console)' required: false @@ -56,7 +57,7 @@ jobs: run: | INPUT="${{ github.event.inputs.vendor || 'all' }}" if [ "$INPUT" = "all" ]; then - echo 'vendors=["digitalocean","vultr"]' >> "$GITHUB_OUTPUT" + echo 'vendors=["digitalocean","vultr","aws"]' >> "$GITHUB_OUTPUT" else echo "vendors=[\"$INPUT\"]" >> "$GITHUB_OUTPUT" fi @@ -105,8 +106,17 @@ jobs: -f reason="$REASON" echo "Dispatched Release Vultr on ref $REF" + dispatch-aws: + needs: resolve-vendors + if: contains(needs.resolve-vendors.outputs.vendors, 'aws') + uses: ./.github/workflows/packer-release-aws.yml + secrets: inherit + with: + version: ${{ needs.resolve-vendors.outputs.version }} + reason: ${{ needs.resolve-vendors.outputs.reason }} + summary: - needs: [resolve-vendors, dispatch-digitalocean, dispatch-vultr] + needs: [resolve-vendors, dispatch-digitalocean, dispatch-vultr, dispatch-aws] if: always() runs-on: ubuntu-latest steps: @@ -121,4 +131,5 @@ jobs: echo "| Reason | ${{ needs.resolve-vendors.outputs.reason }} |" echo "| DigitalOcean | ${{ needs.dispatch-digitalocean.result }} |" echo "| Vultr | ${{ needs.dispatch-vultr.result }} |" + echo "| AWS | ${{ needs.dispatch-aws.result }} |" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 4ee8c45..08a2c58 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ Thumbs.db .secrets .env .ci-ssh-key + +# Python +__pycache__/ +*.pyc +manifest.json diff --git a/deploy/aws/.gitignore b/deploy/aws/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/deploy/aws/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/deploy/aws/AGENTS.md b/deploy/aws/AGENTS.md new file mode 100644 index 0000000..227c177 --- /dev/null +++ b/deploy/aws/AGENTS.md @@ -0,0 +1,117 @@ +# deploy/aws/AGENTS.md + +AWS Marketplace deployment specifics. See `deploy/AGENTS.md` for the platform-agnostic guide. + +## AWS CLI Tools + +### aws (AWS CLI v2) + +- **Install**: Pre-installed on GitHub Actions `ubuntu-latest` runners. Locally: `pip install awscli` or `apt-get install awscli`. +- **Auth**: Reads `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` env vars. For CI with OIDC, use `aws-actions/configure-aws-credentials@v4` with `role-to-assume`. +- **Region**: Always `us-east-1` for marketplace builds. Set via `AWS_DEFAULT_REGION` or `--region`. +- **Output**: `--output json` for machine parsing; default is `json` in CI. +- **AMI listing**: `aws ec2 describe-images --owners self --filters "Name=name,Values=pinner-s3-aws-*"` +- **AMI deregistration**: `aws ec2 deregister-image --image-id ` followed by `aws ec2 delete-snapshot --snapshot-id ` (AMIs leave orphaned snapshots that must be deleted separately) +- **Instance IPs**: `aws ec2 describe-instances --instance-ids --query 'Reservations[0].Instances[0].PublicIpAddress' --output text` +- **Instance termination**: `aws ec2 terminate-instances --instance-ids ` + +### boto3 (Python SDK) + +- The prune scripts (`prune-snapshots.py`, `prune-instances.py`) use boto3. +- **Installation**: `pip install boto3` (available on GitHub Actions runners) +- **Auth**: Same env vars as AWS CLI (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION`) +- **Pagination**: boto3 paginators handle EC2 `DescribeImages` and `DescribeInstances` pagination automatically + +### Marketplace Submission + +- **No direct API**: AWS Marketplace does not expose a public API for creating/updating marketplace listings. Submission is manual via the AWS Marketplace Management Portal (AMMP). +- **AMI sharing**: Before submission, the AMI must be shared with the AWS Marketplace scan account (`679593333241`). `submit.py` prints the exact `aws ec2 modify-image-attribute` command. +- **Manual process**: `submit.py` verifies the AMI exists, prints the sharing command, then outputs the AMMP submission steps: + 1. AMMP -> Products -> Add Product + 2. Restrict AMI to your AWS account + scan account + 3. Submit for scanning (takes ~45 minutes) + 4. Complete product listing metadata + 5. Submit for review + +## AWS Marketplace Requirements + +### Required Scripts (run as final Packer provisioners) + +| Script | Source | Purpose | +|--------|--------|---------| +| `shared/scripts/014-ufw-s3.sh` | Shared | Enable ufw firewall (SSH + port 80, Docker FORWARD policy) | +| `shared/scripts/018-force-ssh-logout.sh` | Shared | Block SSH until `001_onboot` completes first-boot setup | +| `scripts/010-aws-credentials.sh` | AWS-specific | Parse EC2 user-data for S3 credentials and auto-update flags (runs before `001_onboot`) | +| `scripts/900-cleanup.sh` | AWS-specific | Clear logs, SSH keys, bash history, zero disk, machine-id, remove authorized keys, fstrim | + +### Cloud-init Boot Sequence + +Cloud-init executes `per-instance` scripts in lexicographic order: + +1. `010-aws-credentials.sh` — Parses EC2 user-data for `s3_access_key`, `s3_secret_key`, `auto_update`. Writes `/opt/s3-server/.env` and `/state/autoupdate.{enabled,disabled}`. +2. `001_onboot` (shared) — Removes SSH lockout, enables ufw, waits for Docker, starts `s3-server.service`. + +### Validation + +AWS does not provide a standard marketplace validation script. The `validate.sh` script creates a temp EC2 instance from the AMI and performs health checks: + +- SSH connectivity (cloud-init completed, SSH lockout removed) +- `s3-server.service` is active +- Docker containers are running +- HTTP healthz endpoint responds with 200 +- AWS-specific compliance: `PermitRootLogin no`, `PasswordAuthentication no`, no `ubuntu` authorized keys, no host keys + +**SSH wait**: 300s (30 attempts × 10s). AWS instances typically boot within 60-90s. + +**Healthcheck**: The Docker healthcheck in `docker-compose.yml` must use `wget -q -O /dev/null` (GET), NOT `wget --spider` (HEAD). The `/_panel/healthz` endpoint is registered as GET-only in Echo, so HEAD returns 405 Method Not Allowed. + +### Marketplace Compliance (FAIL = rejection) + +- Supported OS: Ubuntu 22.04 LTS (AMI built from official Ubuntu AMI) +- AMI must be HVM, 64-bit, EBS-backed +- Built in `us-east-1` +- cloud-init installed +- No root password set +- No SSH keys in `/root/.ssh/` or `/home/ubuntu/.ssh/` +- Root and user bash history cleared +- No AWS Systems Manager agent or CloudWatch agent pre-installed +- No pending security updates +- Firewall configured (ufw or security groups) +- `PermitRootLogin no` and `PasswordAuthentication no` in `/etc/ssh/sshd_config` +- IMDSv2 compliant (optional but recommended) +- Logs cleared from `/var/log` + +## Required Secrets + +| Secret | Purpose | +|--------|---------| +| `AWS_ACCESS_KEY_ID` | Packer builds, AWS CLI auth, boto3 auth | +| `AWS_SECRET_ACCESS_KEY` | Packer builds, AWS CLI auth, boto3 auth | +| `CI_BUILD_SSH_KEY` | Private SSH key (ed25519, base64-encoded). Written to `~/.ssh/id_ed25519` on the runner. Used for Packer SSH access and validation instance SSH. | + +## AWS-Specific Configuration + +### Packer Builder + +| Field | Value | Notes | +|-------|-------|-------| +| `source_ami_filter` | Ubuntu 22.04 LTS official AMI | `name: "ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"`, `owners: ["099720109477"]` (Canonical) | +| `instance_type` | `t3.small` | 2 vCPU, 2GB RAM. Required for Docker provisioning. | +| `region` | `us-east-1` | AWS Marketplace requires AMIs to be built in us-east-1. | +| `ssh_username` | `ubuntu` | Official Ubuntu AMIs use `ubuntu` user. | +| `ami_name` | `pinner-s3-aws-` | Timestamp from Packer `{{timestamp}}`. | +| `ena_support` | `true` | Required for modern instance types. | +| `sriov_support` | `true` | Required for enhanced networking. | +| `ssh_interface` | `public_ip` | Packer connects via public IP. | + +### AWS-Specific Pitfalls + +- **AMI deregistration leaves snapshots**: `aws ec2 deregister-image` does NOT delete the underlying EBS snapshot. You must call `aws ec2 delete-snapshot` separately or use `prune-snapshots.py` which handles both. +- **Security group ingress**: The CloudFormation template defaults to `0.0.0.0/0` on port 80 and 22. End users should restrict this. The template documents this in parameter descriptions. +- **No marketplace submit API**: The `submit.py` script verifies the AMI exists and prints the sharing command + AMMP steps. Cannot programmatically submit. +- **IMDSv2**: AWS recommends requiring IMDSv2. The Packer template sets `metadata_options { http_tokens = "optional" }` to allow both v1 and v2 during build. The CloudFormation template sets `MetadataOptions` with `HttpTokens: optional` by default; users can override. +- **AMI copy for other regions**: Marketplace listings can copy the AMI to other regions after us-east-1 submission. The initial build must stay in us-east-1. +- **Prune scripts**: Split into `prune-snapshots.py` (AMIs + snapshots) and `prune-instances.py` (EC2 instances). Each requires `--prefix` (no default). Never combine snapshot and instance deletion. +- **SSH user**: Official Ubuntu AMIs use `ubuntu`, not `root`. The Packer `ssh_username` is `ubuntu`. Cleanup must clear `/home/ubuntu/.ssh/authorized_keys` in addition to `/root/.ssh/`. +- **User-data parsing**: EC2 user-data is plain text. `010-aws-credentials.sh` uses `sed` with strict regex to extract key=value pairs. It never evaluates user-data as shell code (security requirement). +- **CloudFormation template**: `ImageId` uses `AWS::EC2::Image::Id` type (not plain `String`) so the AWS Console presents an AMI picker. Sensitive parameters use `NoEcho: true`. diff --git a/deploy/aws/MARKETPLACE_README.md b/deploy/aws/MARKETPLACE_README.md new file mode 100644 index 0000000..0cc8798 --- /dev/null +++ b/deploy/aws/MARKETPLACE_README.md @@ -0,0 +1,32 @@ +# AWS Marketplace Listing + +## App Name +Pinner S3 Server + +## Short Description +Self-hosted, private S3-compatible object storage with automatic updates on AWS. + +## Description +Pinner S3 Server is a lightweight, self-hosted S3-compatible object storage service that runs in Docker on Amazon EC2. It includes a built-in update sidecar that automatically checks for and applies new container image releases. Launch the AMI or deploy the CloudFormation template to start managing your own object storage. + +### Features +- S3-compatible API on port 80 +- Automatic container image updates (ON by default) +- Docker Compose-based deployment for easy management +- Health checks and auto-restart on failure +- CloudFormation template with security-group guidance +- IPv4 and IPv6 ready ( listening port is bound for both) + +### Getting Started +1. Subscribe to the product in AWS Marketplace and launch the CloudFormation stack (or AMI directly). +2. The service starts automatically on first boot and pulls the latest container images. +3. Access the S3-compatible API at `http://your-ec2-ip/`. +4. Configure credentials by passing `s3_access_key` and `s3_secret_key` as EC2 user-data or by editing `/opt/s3-server/.env`. + +### Support +For documentation and support, visit https://pinner.xyz. + +## Product Categories +- Storage +- Cloud Infrastructure +- Developer Tools diff --git a/deploy/aws/Makefile b/deploy/aws/Makefile new file mode 100644 index 0000000..373f88d --- /dev/null +++ b/deploy/aws/Makefile @@ -0,0 +1,63 @@ +.PHONY: build validate submit cleanup-snapshot prune prune-snapshots prune-instances all clean + +TEMPLATE := template.pkr.hcl +MANIFEST := manifest.json +DRY_RUN ?= true +VERSION ?= +REASON ?= Automated release +SNAPSHOT_PREFIX ?= pinner-s3-aws- + +all: build validate submit + +VPC_ID ?= +SUBNET_ID ?= + +build: + packer init . + packer validate . + packer build -force \ + -var "snapshot_prefix=$(SNAPSHOT_PREFIX)" \ + $(if $(VPC_ID),-var "vpc_id=$(VPC_ID)",) \ + $(if $(SUBNET_ID),-var "subnet_id=$(SUBNET_ID)",) \ + . + +validate: + @command -v aws >/dev/null 2>&1 || { echo "Error: aws CLI is required for validation"; exit 1; } + @command -v jq >/dev/null 2>&1 || { echo "Error: jq is required for validation"; exit 1; } + bash ./scripts/validate.sh $(MANIFEST) + +submit: + python3 submit.py --manifest $(MANIFEST) --dry-run=$(DRY_RUN) --version="$(VERSION)" --reason="$(REASON)" + +cleanup-snapshot: + @command -v aws >/dev/null 2>&1 || { echo "Error: aws CLI is required for cleanup"; exit 1; } + @command -v jq >/dev/null 2>&1 || { echo "Error: jq is required for cleanup"; exit 1; } + @if [ ! -f $(MANIFEST) ]; then echo "No manifest found, nothing to clean up"; exit 0; fi + @AMI_ID=$$(jq -r '.builds[-1].artifact_id' $(MANIFEST) | sed 's/^[^:]*://') && \ + if [ -z "$$AMI_ID" ] || [ "$$AMI_ID" = "null" ]; then echo "No AMI ID in manifest"; exit 0; fi && \ + echo "==> Deregistering AMI $$AMI_ID..." && \ + aws ec2 deregister-image --image-id "$$AMI_ID" && \ + for SNAP in $$(aws ec2 describe-images --image-ids "$$AMI_ID" --query 'Images[0].BlockDeviceMappings[*].Ebs.SnapshotId' --output text); do \ + if [ -n "$$SNAP" ] && [ "$$SNAP" != "None" ]; then \ + echo "==> Deleting snapshot $$SNAP..."; \ + aws ec2 delete-snapshot --snapshot-id "$$SNAP"; \ + fi; \ + done && \ + echo "==> AMI and snapshots cleaned up" || { echo "Warning: failed to clean up AMI $$AMI_ID"; exit 1; } + +clean: + rm -f $(MANIFEST) + +prune: prune-snapshots prune-instances + +prune-snapshots: + MAX_AGE=$${MAX_AGE_HOURS:-24}; \ + ARGS="--max-age-hours $$MAX_AGE --prefix pinner-s3-aws-"; \ + if [ "$${DRY_RUN:-true}" = "true" ]; then ARGS="$$ARGS --dry-run"; fi; \ + python3 scripts/prune-snapshots.py $$ARGS + +prune-instances: + MAX_AGE=$${MAX_AGE_HOURS:-24}; \ + ARGS="--max-age-hours $$MAX_AGE --prefix pinner-s3-aws-"; \ + if [ "$${DRY_RUN:-true}" = "true" ]; then ARGS="$$ARGS --dry-run"; fi; \ + python3 scripts/prune-instances.py $$ARGS diff --git a/deploy/aws/README.md b/deploy/aws/README.md new file mode 100644 index 0000000..2c581e5 --- /dev/null +++ b/deploy/aws/README.md @@ -0,0 +1,107 @@ +# Pinner S3 Server: AWS Marketplace Deployment + +Deploy [S3 Server](https://github.com/lumeweb/s3-server) on AWS EC2 via CloudFormation using a pre-built Packer AMI with auto-update sidecar. + +## What This Deploys + +| Component | Description | +|-----------|-------------| +| **s3-server** | Object storage server (`ghcr.io/lumeweb/s3-server:latest`) on port **80** | +| **updater** | Sidecar container (`ghcr.io/lumeweb/s3-server-updater:latest`) that polls GHCR for new images every 6 hours | +| **Root EBS Volume** | Single gp3 root volume (`/dev/sda1`, default 50GB) holds OS, Docker images, and `/data` | +| **Elastic IP** | Static public IP assigned to the instance | +| **Security Group** | Inbound: TCP 80 (app), optional TCP 22 (SSH); Outbound: all | +| **Systemd** | `s3-server.service` starts Docker Compose on boot | + +## Requirements + +- **AWS account** with VPC and subnet access +- Packer AMI published to AWS Marketplace +- IAM permissions: EC2, EBS, Elastic IP, CloudFormation, Security Groups + +## Quick Start + +### Option A: CloudFormation Stack (Recommended) + +1. Go to **AWS CloudFormation Console** -> **Create Stack** -> **With new resources (standard)** +2. Upload `deploy/aws/cloudformation.template.json` or reference it via S3 URL +3. Fill parameters: + - **VpcId**: Your VPC ID + - **SubnetId**: Target subnet for the instance + - **S3ServerAMI**: The S3 Server marketplace AMI ID (region-specific) + - **InstanceType**: `t3.small` (default) or higher + - **VolumeSize**: 50 (default, min 20GB) + - **AutoUpdate**: `true` (default) or `false` + - **AllowedAppCIDR**: `0.0.0.0/0` (default) or restrict + - **AllowedSSHCIDR**: Your trusted IP range, or leave blank to disable SSH + - **S3AccessKey / S3SecretKey**: Optional S3 credentials (passed via EC2 user-data) +4. Click **Create Stack** +5. Check the **Outputs** tab for `S3ServerURL` (e.g., `http://x.x.x.x/`) + +### Option B: Build the AMI with Packer + +```bash +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +cd /path/to/s3-cloud +cd deploy/aws +make build +``` + +This builds `pinner-s3-aws-` based on Ubuntu 22.04 LTS. + +## Persistent Storage + +The CloudFormation template uses a single gp3 root volume (`/dev/sda1`, default 50GB, min 20GB) for the OS, Docker images, and the `/data` directory mounted into the s3-server container. + +To resize: update the `VolumeSize` parameter and run a CloudFormation stack update, then extend the filesystem with `growpart`/`resize2fs` inside the instance. + +## Auto-Update + +Auto-update is **enabled by default** when `AutoUpdate=true`. The updater sidecar container checks GHCR for new `:latest` image digests every 6 hours and recreates the `s3-server` container when a new digest is found. + +The EC2 user-data passes `auto_update=true` or `auto_update=false` to the per-instance provisioning script (`scripts/001_provision.sh`), which sets the appropriate flag file in `/state` on first boot. + +### Flag-File API + +| File | Purpose | +|------|---------| +| `/state/autoupdate.enabled` | Present = auto-update ON | +| `/state/autoupdate.disabled` | Present = auto-update OFF (takes precedence) | +| `/state/update.trigger` | Force update check on next cycle | +| `/state/updater.log` | Append-only updater log | +| `/state/last-digest` | Last applied image digest | + +### Disable Auto-Update + +```bash +ssh ubuntu@ +sudo touch /state/autoupdate.disabled +sudo docker compose -f /opt/s3-server/docker-compose.yml restart +``` + +### Force a Manual Update + +```bash +sudo touch /state/update.trigger +``` + +## Files + +| File | Purpose | +|------|---------| +| `cloudformation.template.json` | CloudFormation stack template for EC2 deployment | +| `template.pkr.hcl` | Packer template to build the AWS Marketplace AMI (x86-64) | +| `scripts/001_provision.sh` | Cloud-init per-instance script: reads EC2 user-data for S3 credentials and auto-update config | +| `scripts/900-cleanup.sh` | Final Packer provisioner: hardens and cleans the AMI | +| `scripts/validate.sh` | Creates a temp EC2 instance from the AMI and runs compliance/health checks | +| `scripts/prune-snapshots.py` | Deregisters old CI AMIs and deletes their snapshots | +| `scripts/prune-instances.py` | Terminates old CI EC2 instances | +| `submit.py` | Prints AMI ID and AWS Marketplace sharing instructions | +| `marketplace/` | Submission assets required by AWS Marketplace | +| `README.md` | This file | + +## License + +This deployment configuration is provided under the MIT License. +The S3 Server itself is licensed separately - see the [upstream repo](https://github.com/lumeweb/s3-server). diff --git a/deploy/aws/cloudformation.template.json b/deploy/aws/cloudformation.template.json new file mode 100644 index 0000000..469659c --- /dev/null +++ b/deploy/aws/cloudformation.template.json @@ -0,0 +1,213 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Pinner S3 Server - Self-hosted S3-compatible object storage. Deploys an EC2 instance from a Marketplace AMI with optional auto-update and S3 credentials.", + "Metadata": { + "AWS::CloudFormation::Interface": { + "ParameterGroups": [ + { + "Label": { "default": "Network Configuration" }, + "Parameters": ["VpcId", "SubnetId", "AllowedSSHCIDR", "AllowedAppCIDR"] + }, + { + "Label": { "default": "Instance Configuration" }, + "Parameters": ["InstanceType", "VolumeSize", "PinnerAMI", "AutoUpdate"] + }, + { + "Label": { "default": "S3 Credentials (Optional)" }, + "Parameters": ["S3AccessKey", "S3SecretKey"] + } + ], + "ParameterLabels": { + "VpcId": { "default": "VPC ID" }, + "SubnetId": { "default": "Subnet ID" }, + "AllowedSSHCIDR": { "default": "Allowed SSH CIDR" }, + "AllowedAppCIDR": { "default": "Allowed Application CIDR" }, + "InstanceType": { "default": "Instance Type" }, + "VolumeSize": { "default": "Root Volume Size (GB)" }, + "PinnerAMI": { "default": "Pinner S3 Server AMI ID" }, + "AutoUpdate": { "default": "Enable Auto-Update" }, + "S3AccessKey": { "default": "S3 Access Key" }, + "S3SecretKey": { "default": "S3 Secret Key" } + } + } + }, + "Parameters": { + "InstanceType": { + "Description": "EC2 instance type for Pinner S3 Server", + "Type": "String", + "Default": "t3.small", + "AllowedValues": [ + "t3.micro", + "t3.small", + "t3.medium", + "t3.large", + "t3.xlarge", + "t3.2xlarge" + ] + }, + "VolumeSize": { + "Description": "Size of the root EBS volume in GB (holds OS, Docker images, and /data)", + "Type": "Number", + "Default": 50, + "MinValue": 20, + "MaxValue": 16384 + }, + "S3AccessKey": { + "Description": "S3 access key (optional, leave blank to run without credentials)", + "Type": "String", + "Default": "", + "NoEcho": true + }, + "S3SecretKey": { + "Description": "S3 secret key (optional, leave blank to run without credentials)", + "Type": "String", + "Default": "", + "NoEcho": true + }, + "VpcId": { + "Description": "ID of the VPC to deploy into", + "Type": "AWS::EC2::VPC::Id" + }, + "SubnetId": { + "Description": "ID of the subnet to deploy into", + "Type": "AWS::EC2::Subnet::Id" + }, + "AllowedSSHCIDR": { + "Description": "CIDR block allowed to access SSH (port 22). Leave blank to disable SSH access entirely (use SSM Session Manager instead).", + "Type": "String", + "Default": "", + "AllowedPattern": "^$|^(\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2}$" + }, + "AllowedAppCIDR": { + "Description": "CIDR block allowed to access the Pinner S3 Server application (port 80). Default 0.0.0.0/0 allows access from anywhere. Restrict to your trusted network for better security.", + "Type": "String", + "Default": "0.0.0.0/0", + "AllowedPattern": "^(\\d{1,3}\\.){3}\\d{1,3}/\\d{1,2}$" + }, + "AutoUpdate": { + "Description": "Enable auto-update sidecar. When enabled, the instance will periodically pull new container images from ghcr.io/lumeweb/s3-server:latest (a seller-controlled registry). Disable to pin the current image version.", + "Type": "String", + "Default": "true", + "AllowedValues": ["true", "false"] + }, + "PinnerAMI": { + "Description": "AMI ID for the Pinner S3 Server marketplace image", + "Type": "AWS::EC2::Image::Id" + } + }, + "Conditions": { + "HasS3Credentials": { + "Fn::And": [ + { "Fn::Not": [{ "Fn::Equals": [{ "Ref": "S3AccessKey" }, ""] }] }, + { "Fn::Not": [{ "Fn::Equals": [{ "Ref": "S3SecretKey" }, ""] }] } + ] + }, + "HasSSHCIDR": { + "Fn::Not": [{ "Fn::Equals": [{ "Ref": "AllowedSSHCIDR" }, ""] }] + } + }, + "Resources": { + "PinnerSecurityGroup": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "Security group for Pinner S3 Server", + "VpcId": { "Ref": "VpcId" }, + "SecurityGroupIngress": [ + { + "IpProtocol": "tcp", + "FromPort": 80, + "ToPort": 80, + "CidrIp": { "Ref": "AllowedAppCIDR" } + }, + { + "Fn::If": [ + "HasSSHCIDR", + { + "IpProtocol": "tcp", + "FromPort": 22, + "ToPort": 22, + "CidrIp": { "Ref": "AllowedSSHCIDR" } + }, + { "Ref": "AWS::NoValue" } + ] + } + ], + "SecurityGroupEgress": [ + { + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + ] + } + }, + "PinnerInstance": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": { "Ref": "PinnerAMI" }, + "InstanceType": { "Ref": "InstanceType" }, + "SubnetId": { "Ref": "SubnetId" }, + "SecurityGroupIds": [{ "Ref": "PinnerSecurityGroup" }], + "UserData": { + "Fn::Base64": { + "Fn::Join": [ + "\n", + [ + "#!/bin/bash", + { "Fn::If": ["HasS3Credentials", { "Fn::Sub": "export s3_access_key=\"${S3AccessKey}\"" }, ""] }, + { "Fn::If": ["HasS3Credentials", { "Fn::Sub": "export s3_secret_key=\"${S3SecretKey}\"" }, ""] }, + { "Fn::Sub": "export auto_update=${AutoUpdate}" } + ] + ] + } + }, + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "VolumeSize": { "Ref": "VolumeSize" }, + "VolumeType": "gp3", + "DeleteOnTermination": true + } + } + ], + "Tags": [ + { "Key": "Name", "Value": "pinner-s3-server" }, + { "Key": "Project", "Value": "pinner" } + ] + } + }, + "PinnerEIP": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc" + } + }, + "PinnerEIPAssociation": { + "Type": "AWS::EC2::EIPAssociation", + "Properties": { + "InstanceId": { "Ref": "PinnerInstance" }, + "AllocationId": { "Fn::GetAtt": ["PinnerEIP", "AllocationId"] } + } + } + }, + "Outputs": { + "PinnerURL": { + "Description": "URL to access Pinner S3 Server", + "Value": { + "Fn::Join": ["", ["http://", { "Fn::GetAtt": ["PinnerEIP", "PublicIp"] }]] + } + }, + "InstanceId": { + "Description": "EC2 instance ID", + "Value": { "Ref": "PinnerInstance" } + }, + "PublicIP": { + "Description": "Public IP address of Pinner S3 Server", + "Value": { "Fn::GetAtt": ["PinnerEIP", "PublicIp"] } + }, + "AllocationId": { + "Description": "Elastic IP allocation ID", + "Value": { "Fn::GetAtt": ["PinnerEIP", "AllocationId"] } + } + } +} diff --git a/deploy/aws/iam-policy.json b/deploy/aws/iam-policy.json new file mode 100644 index 0000000..6fd62dd --- /dev/null +++ b/deploy/aws/iam-policy.json @@ -0,0 +1,112 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "PackerAMI", + "Effect": "Allow", + "Action": [ + "ec2:DescribeImages", + "ec2:RegisterImage", + "ec2:DeregisterImage", + "ec2:CreateImage", + "ec2:DescribeImageAttribute", + "ec2:ModifyImageAttribute", + "ec2:ResetImageAttribute", + "ec2:CreateTags", + "ec2:DescribeTags" + ], + "Resource": "*" + }, + { + "Sid": "PackerSnapshots", + "Effect": "Allow", + "Action": [ + "ec2:CreateSnapshot", + "ec2:DeleteSnapshot", + "ec2:DescribeSnapshots", + "ec2:ModifySnapshotAttribute", + "ec2:ResetSnapshotAttribute" + ], + "Resource": "*" + }, + { + "Sid": "PackerInstances", + "Effect": "Allow", + "Action": [ + "ec2:RunInstances", + "ec2:TerminateInstances", + "ec2:DescribeInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeInstanceAttribute", + "ec2:DescribeInstanceTypes", + "ec2:StopInstances", + "ec2:StartInstances", + "ec2:GetPasswordData" + ], + "Resource": "*" + }, + { + "Sid": "PackerVolumes", + "Effect": "Allow", + "Action": [ + "ec2:CreateVolume", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:AttachVolume", + "ec2:DescribeVolumes", + "ec2:DescribeVolumeStatus", + "ec2:ModifyVolumeAttribute" + ], + "Resource": "*" + }, + { + "Sid": "PackerNetworking", + "Effect": "Allow", + "Action": [ + "ec2:DescribeRegions", + "ec2:DescribeVpcs", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:CreateSecurityGroup", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeAvailabilityZones" + ], + "Resource": "*" + }, + { + "Sid": "PackerKeys", + "Effect": "Allow", + "Action": [ + "ec2:DescribeKeyPairs", + "ec2:CreateKeyPair", + "ec2:DeleteKeyPair", + "ec2:ImportKeyPair" + ], + "Resource": "*" + }, + { + "Sid": "PassRole", + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:PassedToService": "ec2.amazonaws.com" + } + } + }, + { + "Sid": "SerialConsole", + "Effect": "Allow", + "Action": [ + "ec2:SerialConsoleAccess", + "ec2:GetSerialConsole", + "ec2:SendSerialConsoleSSHPublicKey" + ], + "Resource": "*" + } + ] +} diff --git a/deploy/aws/marketplace/README.md b/deploy/aws/marketplace/README.md new file mode 100644 index 0000000..1c1d047 --- /dev/null +++ b/deploy/aws/marketplace/README.md @@ -0,0 +1,9 @@ +# AWS Marketplace Assets + +This directory contains the non-code documents and checklists required to submit the Pinner.xyz S3 Server AMI + CloudFormation product to AWS Marketplace. + +- `USAGE_INSTRUCTIONS.md` — buyer-facing post-deployment instructions. +- `SECURITY_GROUPS.md` — recommended inbound/outbound rules and security guidance. +- `SUBMISSION_CHECKLIST.md` — pre-submission and post-submission verification list. + +For the technical deployment files, see the parent `deploy/aws/` directory. diff --git a/deploy/aws/marketplace/SECURITY_GROUPS.md b/deploy/aws/marketplace/SECURITY_GROUPS.md new file mode 100644 index 0000000..259c288 --- /dev/null +++ b/deploy/aws/marketplace/SECURITY_GROUPS.md @@ -0,0 +1,18 @@ +# Recommended Security Group Rules + +## Inbound + +| Protocol | Port | Source | Purpose | +|----------|------|--------|---------| +| TCP | 80 | `AllowedAppCIDR` parameter (default `0.0.0.0/0`) | S3-compatible HTTP API | +| TCP | 22 | `AllowedSSHCIDR` parameter (default empty/disabled) | Administrative SSH access | + +## Outbound + +| Protocol | Port | Destination | Purpose | +|----------|------|-------------|---------| +| All | All | `0.0.0.0/0` | Docker image pulls, apt updates, GHCR polling | + +## Recommendation + +Restrict `AllowedAppCIDR` to your trusted IP ranges or corporate network. Leave `AllowedSSHCIDR` blank and use AWS Systems Manager Session Manager for administrative access. diff --git a/deploy/aws/marketplace/SUBMISSION_CHECKLIST.md b/deploy/aws/marketplace/SUBMISSION_CHECKLIST.md new file mode 100644 index 0000000..aab4e1a --- /dev/null +++ b/deploy/aws/marketplace/SUBMISSION_CHECKLIST.md @@ -0,0 +1,32 @@ +# AWS Marketplace Submission Checklist — Pinner.xyz S3 Server + +## Seller prerequisites +- [ ] AWS Marketplace seller registration complete. +- [ ] Access to AWS Marketplace Management Portal (AMMP). + +## AMI requirements +- [ ] AMI built from `deploy/aws/template.pkr.hcl` in `us-east-1` and owned by seller account. +- [ ] AMI uses HVM virtualization, 64-bit architecture, EBS-backed root device. +- [ ] `PasswordAuthentication no` and `PermitRootLogin no` enforced in `sshd_config`. +- [ ] Authorized keys removed; SSH host keys removed; cloud-init logs cleaned. +- [ ] AMI passes AMMP "Test 'Add Version"" scan with no critical findings. + +## CloudFormation template requirements +- [ ] Template uploaded to AMMP as part of the AMI with CloudFormation delivery option. +- [ ] No default CIDR allowing public SSH (`AllowedSSHCIDR` defaults to empty). +- [ ] Application CIDR is parameterised (`AllowedAppCIDR`). +- [ ] S3 credentials use `NoEcho: true` and are not echoed in outputs. +- [ ] `ImageId` references a template parameter (`AWS::EC2::Image::Id`). +- [ ] External dependency on GHCR disclosed in usage instructions. + +## Listing content +- [ ] Product title, description, and `Description` field of the AMI match. +- [ ] Usage instructions uploaded (`deploy/aws/marketplace/USAGE_INSTRUCTIONS.md`). +- [ ] Security group recommendations documented (`deploy/aws/marketplace/SECURITY_GROUPS.md`). +- [ ] External dependency (GHCR auto-update) disclosed in usage instructions. +- [ ] Architecture diagram uploaded to S3 and URL provided in listing (if required). + +## Post-submission +- [ ] Product released to **Limited** state; allowlist the test account(s). +- [ ] Launch an instance from the Limited listing and verify end-to-end S3 API access. +- [ ] Request **Public** visibility; Seller Operations review takes 7–10 business days. diff --git a/deploy/aws/marketplace/USAGE_INSTRUCTIONS.md b/deploy/aws/marketplace/USAGE_INSTRUCTIONS.md new file mode 100644 index 0000000..ab1a658 --- /dev/null +++ b/deploy/aws/marketplace/USAGE_INSTRUCTIONS.md @@ -0,0 +1,39 @@ +# Pinner.xyz S3 Server — AWS Marketplace Usage Instructions + +## What is deployed + +Pinner.xyz S3 Server is a private, zero-knowledge, self-hosted S3-compatible object storage server. The AWS Marketplace listing deploys a single Amazon EC2 instance from a pre-built AMI. The AMI includes Docker, Docker Compose, and an auto-update sidecar. + +## Deployment requirements + +- An existing VPC and subnet in the target AWS Region. +- The Marketplace AMI ID for your Region (supplied by the listing). +- (Optional) S3-compatible access credentials. If provided, they are passed through EC2 user-data and written to `/opt/s3-server/.env` on first boot. + +## Post-deployment steps + +1. After the CloudFormation stack reaches `CREATE_COMPLETE`, open the **Outputs** tab and copy the `S3ServerURL` value (`http:///`). +2. Allow 1–2 minutes for the Docker containers to pull and start on first boot. +3. Verify the service: `systemctl status s3-server`. +4. Access the S3-compatible API on port **80**. + +## Auto-update + +The deployment includes an optional auto-update sidecar that polls `ghcr.io/lumeweb/s3-server:latest` every 6 hours and recreates the `s3-server` container when a new image digest is available. This keeps the instance current without manual intervention. The sidecar is enabled by default and can be disabled at launch via the `AutoUpdate` parameter or at runtime by creating `/state/autoupdate.disabled`. + +## External dependencies + +This product requires an internet connection to deploy properly. The following are downloaded or accessed on deployment and on an ongoing basis: + +- Docker Engine and Docker Compose plugin from official Docker repositories. +- Container images from `ghcr.io/lumeweb/s3-server:latest` and `ghcr.io/lumeweb/s3-server-updater:latest`. + +## Security + +- SSH access is disabled by default. To enable SSH, provide a trusted CIDR in the `AllowedSSHCIDR` parameter. We recommend using AWS Systems Manager Session Manager instead. +- The security group allows inbound TCP 80 from the CIDR specified in `AllowedAppCIDR`. Restrict this to your trusted network in production. +- S3 credentials are optional. If supplied, they are marked `NoEcho` in CloudFormation and are not returned in stack outputs. + +## Support + +For documentation, issues, and source code, visit https://github.com/LumeWeb/s3-server. diff --git a/deploy/aws/scripts/000-aws-credentials.sh b/deploy/aws/scripts/000-aws-credentials.sh new file mode 100644 index 0000000..e7caca3 --- /dev/null +++ b/deploy/aws/scripts/000-aws-credentials.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# 010-aws-credentials.sh - AWS-specific cloud-init per-instance script +# Runs before 001_onboot (cloud-init executes scripts in lexicographic order). +# Extracts S3 credentials from EC2 user-data and writes them to /opt/s3-server/.env. +# +# This script is AWS-specific because EC2 user-data is the standard way to pass +# initial configuration to instances. Other vendors use different mechanisms +# (cloud-init vendor-data, metadata service, etc.). + +set -uo pipefail + +ENV_FILE="/opt/s3-server/.env" +mkdir -p /opt/s3-server + +S3_ACCESS_KEY="" +S3_SECRET_KEY="" +AUTO_UPDATE="" + +USER_DATA_FILE="/var/lib/cloud/instance/user-data.txt" +USER_DATA="" + +if [ -f "${USER_DATA_FILE}" ]; then + USER_DATA="$(cat "${USER_DATA_FILE}")" +fi + +# Only extracts specific key=value lines, never evaluates user-data as shell code. +if [ -n "${USER_DATA}" ]; then + S3_ACCESS_KEY="$(echo "${USER_DATA}" | sed -nE 's/^\s*(export\s+)?s3_access_key\s*=\s*"?([^"\n]+?)"?\s*$/\2/p' | head -1 || true)" + S3_SECRET_KEY="$(echo "${USER_DATA}" | sed -nE 's/^\s*(export\s+)?s3_secret_key\s*=\s*"?([^"\n]+?)"?\s*$/\2/p' | head -1 || true)" + AUTO_UPDATE="$(echo "${USER_DATA}" | sed -nE 's/^\s*(export\s+)?auto_update\s*=\s*"?([^"\n]+?)"?\s*$/\2/p' | head -1 || true)" +fi + +if [ -n "${S3_ACCESS_KEY}" ] || [ -n "${S3_SECRET_KEY}" ]; then + echo "[010-aws-credentials] Writing S3 credentials to ${ENV_FILE}" + cat > "${ENV_FILE}" </dev/null || true + rm -f /state/autoupdate.enabled 2>/dev/null || true + echo "[010-aws-credentials] Auto-update disabled via user-data" +elif [ -f /state/autoupdate.disabled ]; then + echo "[010-aws-credentials] Auto-update disabled (flag file present)" +else + touch /state/autoupdate.enabled 2>/dev/null || true + echo "[010-aws-credentials] Auto-update enabled" +fi + +exit 0 diff --git a/deploy/aws/scripts/900-cleanup.sh b/deploy/aws/scripts/900-cleanup.sh new file mode 100755 index 0000000..4965a37 --- /dev/null +++ b/deploy/aws/scripts/900-cleanup.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# 900-cleanup.sh - AWS Marketplace AMI cleanup +# Runs as the final Packer provisioner to prepare the image for submission. + +set -o errexit + +# Ensure /tmp exists and has the proper permissions +if [[ ! -d /tmp ]]; then + mkdir /tmp +fi +chmod 1777 /tmp + +# Wait for any apt/dpkg locks to be released +while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || fuser /var/lib/apt/lists/lock >/dev/null 2>&1; do + sleep 5 +done + +apt-get -y update +DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' upgrade +apt-get -y autoremove +apt-get -y autoclean + +# Clean temporary files +rm -rf /tmp/* /var/tmp/* + +# Remove SSH keys and host keys (regenerated on first boot) +rm -f /root/.ssh/authorized_keys /etc/ssh/*key* +touch /etc/ssh/revoked_keys +chmod 600 /etc/ssh/revoked_keys + +# Clean logs +find /var/log -mtime -1 -type f -exec truncate -s 0 {} \; +rm -rf /var/log/*.gz /var/log/*.[0-9] /var/log/*-???????? +: > /var/log/lastlog +: > /var/log/wtmp + +# Clean cloud-init instance data and logs, but preserve seed for next boot +# --seed forces cloud-init to re-run all modules on the next boot (fresh instance) +cloud-init clean --logs --seed 2>/dev/null || true +rm -rf /var/lib/cloud/instances/* + +# Clean bash history +history -c +: > /root/.bash_history +unset HISTFILE + +# Remove systemd random-seed and machine-id (regenerated on first boot) +rm -f /var/lib/systemd/random-seed +: > /etc/machine-id +[[ -e /var/lib/dbus/machine-id ]] && : > /var/lib/dbus/machine-id + +# Securely erase the unused portion of the filesystem +printf "Writing zeros to remaining disk space (may take several minutes)...\n" +dd if=/dev/zero of=/zerofile bs=4096 || true +rm -f /zerofile +sync + +fstrim / || true diff --git a/deploy/aws/scripts/prune-instances.py b/deploy/aws/scripts/prune-instances.py new file mode 100755 index 0000000..b14e407 --- /dev/null +++ b/deploy/aws/scripts/prune-instances.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Prune old AWS EC2 instances created by CI. + +Instances must have a Name tag matching the prefix and be older than +--max-age-hours. + +Usage: + python3 prune-instances.py --prefix pinner-s3-aws- --max-age-hours 24 +""" + +import argparse +import datetime +import json +import subprocess +import sys + + +def run_aws(*args): + cmd = ["aws", "ec2"] + list(args) + ["--output", "json"] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"aws cli error: {result.stderr}", file=sys.stderr) + sys.exit(1) + return json.loads(result.stdout or "{}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--prefix", required=True) + parser.add_argument("--max-age-hours", type=int, default=24) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + instances = run_aws("describe-instances", "--filters", f"Name=tag:Name,Values={args.prefix}*") + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=args.max_age_hours) + + to_terminate = [] + for res in instances.get("Reservations", []): + for inst in res.get("Instances", []): + state = inst.get("State", {}).get("Name", "") + if state in ("terminated", "shutting-down"): + continue + launch = inst.get("LaunchTime", "") + try: + launch_dt = datetime.datetime.fromisoformat(launch.replace("Z", "+00:00")) + except ValueError: + continue + if launch_dt > cutoff: + continue + name = next((t["Value"] for t in inst.get("Tags", []) if t["Key"] == "Name"), "") + if not name.startswith(args.prefix): + continue + to_terminate.append(inst["InstanceId"]) + print(f"{'Would terminate' if args.dry_run else 'Terminating'} {inst['InstanceId']} ({name}) launched {launch}") + + if args.dry_run or not to_terminate: + return + + subprocess.run(["aws", "ec2", "terminate-instances", "--instance-ids"] + to_terminate, check=True) + + +if __name__ == "__main__": + main() diff --git a/deploy/aws/scripts/prune-snapshots.py b/deploy/aws/scripts/prune-snapshots.py new file mode 100755 index 0000000..6b8bd35 --- /dev/null +++ b/deploy/aws/scripts/prune-snapshots.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Prune old AWS AMIs created by CI. + +AMIs must match the given --prefix and be older than --max-age-hours. +Release AMIs (bare prefix without -pr- or -ci-) are never deleted. + +Usage: + python3 prune-snapshots.py --prefix pinner-s3-aws-pr-123- --max-age-hours 24 +""" + +import argparse +import datetime +import json +import subprocess +import sys + + +def run_aws(*args): + cmd = ["aws", "ec2"] + list(args) + ["--output", "json"] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"aws cli error: {result.stderr}", file=sys.stderr) + sys.exit(1) + return json.loads(result.stdout or "{}") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--prefix", required=True) + parser.add_argument("--max-age-hours", type=int, default=24) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + # Only delete CI/prefixed AMIs, never release AMIs + if "-pr-" not in args.prefix and "-ci-" not in args.prefix: + print(f"Refusing to prune non-CI prefix: {args.prefix}") + sys.exit(1) + + images = run_aws("describe-images", "--owners", "self") + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=args.max_age_hours) + + for img in images.get("Images", []): + name = img.get("Name", "") + if not name.startswith(args.prefix): + continue + created = img.get("CreationDate", "") + try: + created_dt = datetime.datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError: + continue + if created_dt > cutoff: + continue + + print(f"{'Would delete' if args.dry_run else 'Deleting'} AMI {img['ImageId']} ({name}) created {created}") + if args.dry_run: + continue + + # Deregister AMI + run_aws("deregister-image", "--image-id", img["ImageId"]) + # Delete associated snapshots + for bdm in img.get("BlockDeviceMappings", []): + ebs = bdm.get("Ebs", {}) + snap_id = ebs.get("SnapshotId") + if snap_id: + run_aws("delete-snapshot", "--snapshot-id", snap_id) + + +if __name__ == "__main__": + main() diff --git a/deploy/aws/scripts/validate.sh b/deploy/aws/scripts/validate.sh new file mode 100755 index 0000000..35a4dda --- /dev/null +++ b/deploy/aws/scripts/validate.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# validate.sh: Create a temp EC2 instance from the Packer AMI, +# run AWS Marketplace compliance checks, then terminate. +# +# Requires: aws CLI, jq +# Env: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION (or AWS_DEFAULT_REGION) +# CI_BUILD_SSH_KEY: base64-encoded ed25519 private key written to ~/.ssh/id_ed25519 +set -euo pipefail + +# AWS uses ubuntu user, not root +export SSH_USER="ubuntu" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/../../shared/scripts/lib/compliance-checks.sh" + +MANIFEST="${1:-manifest.json}" + +if [ ! -f "$MANIFEST" ]; then + echo "Error: manifest not found: $MANIFEST" >&2 + echo "Run 'make build' first." >&2 + exit 1 +fi + +AMI_ID=$(jq -r '.builds[-1].artifact_id' "$MANIFEST" | sed 's/^[^:]*://') + +if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "null" ]; then + echo "Error: could not extract AMI ID from manifest" >&2 + exit 1 +fi + +AWS_REGION="${AWS_REGION:-${AWS_DEFAULT_REGION:-us-east-1}}" +INSTANCE_LABEL="pinner-s3-aws-validate-$(date +%s)" + +# Default to the smallest instance type for validation +INSTANCE_TYPE="${AWS_VALIDATE_INSTANCE_TYPE:-t3.small}" + +# Look up a subnet in the default VPC, avoiding us-east-1e (no t3.small support) +SUBNET_ID="${AWS_VALIDATE_SUBNET_ID:-}" +if [ -z "$SUBNET_ID" ]; then + SUBNET_INFO=$(aws ec2 describe-subnets \ + --filters "Name=default-for-az,Values=true" \ + --query 'Subnets[*].[SubnetId,AvailabilityZone]' --output text --region "$AWS_REGION") + while read -r sub az; do + if [ "$az" != "us-east-1e" ]; then + SUBNET_ID="$sub" + break + fi + done <<< "$SUBNET_INFO" +fi +if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then + echo "Error: could not determine subnet; set AWS_VALIDATE_SUBNET_ID" >&2 + exit 1 +fi + +SG_ID=$(aws ec2 create-security-group --group-name "$INSTANCE_LABEL" --description "Temporary validation security group for S3 Server" --query 'GroupId' --output text --region "$AWS_REGION") + +# Open port 22 for SSH and port 80 for health checks from current IP +CURRENT_IP=$(curl -fsSL https://checkip.amazonaws.com 2>/dev/null || echo "") +if [ -n "$CURRENT_IP" ]; then + aws ec2 authorize-security-group-ingress \ + --group-id "$SG_ID" \ + --protocol tcp --port 22 --cidr "${CURRENT_IP}/32" \ + --region "$AWS_REGION" >/dev/null + aws ec2 authorize-security-group-ingress \ + --group-id "$SG_ID" \ + --protocol tcp --port 80 --cidr "${CURRENT_IP}/32" \ + --region "$AWS_REGION" >/dev/null +fi + +cleanup() { + if [ -n "${INSTANCE_ID:-}" ]; then + echo "==> Terminating validation instance $INSTANCE_ID..." + aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" --region "$AWS_REGION" >/dev/null 2>&1 || true + echo "==> Waiting for instance to terminate..." + aws ec2 wait instance-terminated --instance-ids "$INSTANCE_ID" --region "$AWS_REGION" >/dev/null 2>&1 || true + fi + if [ -n "${SG_ID:-}" ]; then + echo "==> Deleting temporary security group $SG_ID..." + aws ec2 delete-security-group --group-id "$SG_ID" --region "$AWS_REGION" >/dev/null 2>&1 || true + fi + ssh_close "${INSTANCE_IP:-}" 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +echo "==> Creating validation instance from AMI $AMI_ID in $AWS_REGION..." + +# Build user-data for rescue password if provided +USER_DATA="" +if [ -n "${RESCUE_PASSWORD:-}" ]; then + USER_DATA=$(printf '#cloud-config\npassword: %s\nchpasswd: { expire: False }\nssh_pwauth: True\n' "$RESCUE_PASSWORD" | base64 -w0) + echo "==> Rescue password configured for serial console access" +fi + +RUN_INSTANCE_ARGS=( + --image-id "$AMI_ID" + --instance-type "$INSTANCE_TYPE" + --subnet-id "$SUBNET_ID" + --security-group-ids "$SG_ID" + --query 'Instances[0].InstanceId' + --output text + --region "$AWS_REGION" +) + +if [ -n "$USER_DATA" ]; then + RUN_INSTANCE_ARGS+=(--user-data "$USER_DATA") +fi + +INSTANCE_ID=$(aws ec2 run-instances "${RUN_INSTANCE_ARGS[@]}") + +if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" = "None" ]; then + echo "Error: failed to create validation instance" >&2 + exit 1 +fi + +echo "==> Instance $INSTANCE_ID created. Waiting for status checks..." +aws ec2 wait instance-status-ok --instance-ids "$INSTANCE_ID" --region "$AWS_REGION" + +INSTANCE_IP=$(aws ec2 describe-instances --instance-ids "$INSTANCE_ID" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text --region "$AWS_REGION") + +if [ -z "$INSTANCE_IP" ] || [ "$INSTANCE_IP" = "None" ]; then + echo "Error: could not get public IP for instance $INSTANCE_ID" >&2 + exit 1 +fi + +echo "==> Instance IP: $INSTANCE_IP" +wait_for_ssh "$INSTANCE_IP" 30 || exit 1 + +wait_for_cloud_init "$INSTANCE_IP" 12 + +# --- AWS Marketplace compliance checks --- +echo "" +run_compliance_checks "$INSTANCE_IP" + +# AWS-specific: root login disabled and PasswordAuthentication no +ROOT_LOGIN=$(ssh_run "$INSTANCE_IP" "grep -E '^PermitRootLogin' /etc/ssh/sshd_config 2>/dev/null | awk '{print \$2}' || echo missing") +PASS_AUTH=$(ssh_run "$INSTANCE_IP" "grep -E '^PasswordAuthentication' /etc/ssh/sshd_config 2>/dev/null | awk '{print \$2}' || echo missing") + +if [ "$ROOT_LOGIN" = "no" ]; then + echo " PASS: root login disabled" +else + echo " FAIL: PermitRootLogin is $ROOT_LOGIN (expected no)" + COMPLIANCE_FAIL=1 +fi + +if [ "$PASS_AUTH" = "no" ]; then + echo " PASS: PasswordAuthentication disabled" +else + echo " FAIL: PasswordAuthentication is $PASS_AUTH (expected no)" + COMPLIANCE_FAIL=1 +fi + +# --- Health checks --- +run_health_checks "$INSTANCE_IP" + +if [ "$COMPLIANCE_FAIL" -ne 0 ]; then + echo "Error: marketplace compliance checks failed" >&2 + exit 1 +fi + +if [ "$HEALTH_EXIT" -ne 0 ] && [ "$HTTP_CODE" != "200" ]; then + echo "Error: validation failed (service not active and healthz not 200)" >&2 + exit 1 +fi + +echo "==> All checks passed" diff --git a/deploy/aws/submit.py b/deploy/aws/submit.py new file mode 100755 index 0000000..c8dd8d9 --- /dev/null +++ b/deploy/aws/submit.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""AWS Marketplace AMI submission helper. + +Reads manifest.json (produced by Packer's manifest post-processor), +extracts the AMI ID, and provides instructions for submitting the AMI +to the AWS Marketplace Management Portal (AMMP). + +AWS Marketplace does not expose a public API for product submission. The +actual submission is done manually through AMMP or via the Catalog API +after seller onboarding. + +Usage: + python3 submit.py --manifest manifest.json --version 1.0.0 + python3 submit.py --manifest manifest.json --dry-run=true +""" + +import argparse +import json +import os +import subprocess +import sys + + +def get_ami_id(manifest_path: str) -> str: + with open(manifest_path) as f: + manifest = json.load(f) + + builds = manifest.get("builds", []) + if not builds: + sys.exit("Error: no builds in manifest") + + artifact_id = builds[-1].get("artifact_id", "") + if not artifact_id: + sys.exit("Error: empty artifact_id in manifest") + + # Packer AWS artifact_id includes region prefix like "us-east-1:ami-xxx" + # Strip the region prefix for AWS CLI commands + ami_id = artifact_id.split(":")[-1].strip() + if not ami_id: + sys.exit("Error: empty artifact_id in manifest") + + return ami_id + + +def main(): + parser = argparse.ArgumentParser(description="Submit AMI to AWS Marketplace") + parser.add_argument("--manifest", default="manifest.json", help="Path to Packer manifest.json") + parser.add_argument("--version", default="", help="App version string (e.g. 1.0.0)") + parser.add_argument("--reason", default="Automated release", help="Reason for update") + parser.add_argument("--dry-run", default="true", help="Print instructions without sharing (true/false)") + args = parser.parse_args() + + dry_run = args.dry_run.lower() in ("true", "1", "yes") + + if not args.version: + sys.exit("Error: --version required (e.g. 1.0.0)") + + ami_id = get_ami_id(args.manifest) + print(f"AMI ID: {ami_id}") + + print("\n=== DRY RUN ===") + print(f"AMI: {ami_id}") + print(f"Version: {args.version}") + print(f"Reason: {args.reason}") + + if not dry_run: + # AWS Marketplace service account ID for AMI sharing + mp_account = "679593333241" + print(f"\nTo share this AMI with AWS Marketplace, run:") + print(f" aws ec2 modify-image-attribute \\") + print(f" --image-id {ami_id} \\") + print(f" --launch-permission \"Add=[{{UserId={mp_account}}}]\"") + print(f"\nAfter sharing, submit the AMI via the AWS Marketplace Management Portal.") + else: + print("\nDry run: no API calls made.") + print("Set --dry-run=false to print the sharing command.") + + +if __name__ == "__main__": + main() diff --git a/deploy/aws/template.pkr.hcl b/deploy/aws/template.pkr.hcl new file mode 100644 index 0000000..6b8dc2a --- /dev/null +++ b/deploy/aws/template.pkr.hcl @@ -0,0 +1,211 @@ +packer { + required_plugins { + amazon = { + source = "github.com/hashicorp/amazon" + version = ">= 1.2.8" + } + } +} + +variable "aws_region" { + type = string + default = "us-east-1" +} + +variable "aws_access_key" { + type = string + default = env("AWS_ACCESS_KEY_ID") + sensitive = true +} + +variable "aws_secret_key" { + type = string + default = env("AWS_SECRET_ACCESS_KEY") + sensitive = true +} + +variable "aws_session_token" { + type = string + default = env("AWS_SESSION_TOKEN") + sensitive = true +} + +variable "snapshot_prefix" { + type = string + default = "pinner-s3-aws-" +} + +variable "vpc_id" { + type = string + default = "" +} + +variable "subnet_id" { + type = string + default = "" +} + +variable "application_name" { + type = string + default = "Pinner S3 Server" +} + +variable "application_version" { + type = string + default = "1.0.0" +} + +source "amazon-ebs" "s3-server" { + region = var.aws_region + access_key = var.aws_access_key + secret_key = var.aws_secret_key + token = var.aws_session_token + + source_ami_filter { + filters = { + name = "ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + owners = ["099720109477"] # Canonical + most_recent = true + } + + instance_type = "t3.small" + ssh_username = "ubuntu" + ami_name = "${var.snapshot_prefix}{{timestamp}}" + ami_description = "Pinner S3 Server with Docker, docker-compose, and auto-update sidecar (x86-64)" + + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 20 + volume_type = "gp3" + delete_on_termination = true + } + + vpc_id = var.vpc_id != "" ? var.vpc_id : null + subnet_id = var.subnet_id != "" ? var.subnet_id : null + + tags = { + Name = "pinner-s3-server" + Architecture = "x86-64" + Project = "pinner" + Description = "Pinner S3 Server Marketplace AMI (x86-64)" + } + + snapshot_tags = { + Name = "pinner-s3-aws-snapshot" + } +} + +build { + sources = ["source.amazon-ebs.s3-server"] + + # Wait for cloud-init to finish before provisioning + provisioner "shell" { + inline = ["cloud-init status --wait"] + } + + # System updates before installing anything + provisioner "shell" { + environment_vars = [ + "DEBIAN_FRONTEND=noninteractive", + "LC_ALL=C", + "LANG=en_US.UTF-8", + "LC_CTYPE=en_US.UTF-8", + ] + inline = [ + "sudo apt-get -qqy update", + "sudo apt-get -qqy -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' upgrade", + "sudo apt-get -qqy clean", + ] + } + + # Upload docker-compose.yml for the shared provisioner + provisioner "file" { + source = "${path.root}/../../docker-compose.yml" + destination = "/tmp/docker-compose.yml" + } + + # Upload MOTD (shared) to /tmp first, then sudo mv to system path + provisioner "file" { + source = "${path.root}/../shared/files/etc/update-motd.d/99-one-click" + destination = "/tmp/99-one-click" + } + + # Upload cloud-init per-instance boot script (shared) to /tmp first + provisioner "file" { + source = "${path.root}/../shared/files/var/lib/cloud/scripts/per-instance/001_onboot" + destination = "/tmp/001_onboot" + } + + # Upload AWS-specific credential parser to /tmp first + provisioner "file" { + source = "scripts/000-aws-credentials.sh" + destination = "/tmp/000-aws-credentials.sh" + } + + # Move files to system paths with sudo and set executable bits + provisioner "shell" { + inline = [ + "sudo mkdir -p /etc/update-motd.d /var/lib/cloud/scripts/per-instance", + "sudo mv /tmp/99-one-click /etc/update-motd.d/99-one-click", + "sudo mv /tmp/001_onboot /var/lib/cloud/scripts/per-instance/001_onboot", + "sudo mv /tmp/000-aws-credentials.sh /var/lib/cloud/scripts/per-instance/000-aws-credentials.sh", + "sudo chmod +x /etc/update-motd.d/99-one-click", + "sudo chmod +x /var/lib/cloud/scripts/per-instance/001_onboot", + "sudo chmod +x /var/lib/cloud/scripts/per-instance/000-aws-credentials.sh", + ] + } + + # Run the shared provisioner (Docker install, compose, systemd, .env) + # Use sudo since AWS provisioner connects as 'ubuntu' user, not root + provisioner "shell" { + execute_command = "sudo bash -c '{{ .Vars }} {{ .Path }}'" + environment_vars = [ + "DEBIAN_FRONTEND=noninteractive", + "LC_ALL=C", + "LANG=en_US.UTF-8", + "LC_CTYPE=en_US.UTF-8", + ] + script = "${path.root}/../../packer/scripts/install-s3-server.sh" + } + + # Configure firewall and SSH lockout (shared) + # Use sudo since AWS provisioner connects as 'ubuntu' user, not root + provisioner "shell" { + execute_command = "sudo bash -c '{{ .Vars }} {{ .Path }}'" + environment_vars = [ + "DEBIAN_FRONTEND=noninteractive", + "LC_ALL=C", + "LANG=en_US.UTF-8", + "LC_CTYPE=en_US.UTF-8", + ] + scripts = [ + "${path.root}/../shared/scripts/014-ufw-s3.sh", + "${path.root}/../shared/scripts/018-force-ssh-logout.sh", + ] + } + + # AWS Marketplace hardening: disable password auth and lock root login + provisioner "shell" { + execute_command = "sudo bash -c '{{ .Vars }} {{ .Path }}'" + inline = [ + "sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config || echo 'PasswordAuthentication no' >> /etc/ssh/sshd_config", + "sed -i 's/^#*PermitRootLogin.*/PermitRootLogin no/' /etc/ssh/sshd_config || echo 'PermitRootLogin no' >> /etc/ssh/sshd_config", + "systemctl restart sshd || true", + ] + } + + # Run cleanup script last + provisioner "shell" { + execute_command = "sudo bash -c '{{ .Vars }} {{ .Path }}'" + script = "scripts/900-cleanup.sh" + } + + # Output manifest.json with AMI ID for release automation + post-processor "manifest" { + output = "manifest.json" + strip_path = true + } +} diff --git a/deploy/shared/scripts/lib/compliance-checks.sh b/deploy/shared/scripts/lib/compliance-checks.sh index 7af19d9..0e90165 100644 --- a/deploy/shared/scripts/lib/compliance-checks.sh +++ b/deploy/shared/scripts/lib/compliance-checks.sh @@ -8,8 +8,15 @@ # so no per-vendor SSH configuration is needed. SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519}" +SSH_USER="${SSH_USER:-root}" SSH_BASE_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY") +# Remote host spec (user@ip) +_ssh_host() { + local ip="$1" + echo "${SSH_USER}@${ip}" +} + # Global: set to 1 if any compliance check fails. Validate scripts check this # after run_compliance_checks() returns. # shellcheck disable=SC2034 @@ -28,7 +35,7 @@ _ssh_open() { local socket socket=$(_ssh_socket_path "$ip") - if ssh -O check -o ControlPath="$socket" "root@$ip" 2>/dev/null; then + if ssh -O check -o ControlPath="$socket" "$(_ssh_host "$ip")" 2>/dev/null; then return 0 fi @@ -37,11 +44,11 @@ _ssh_open() { -o ControlPath="$socket" \ -o ControlPersist=600 \ "${SSH_BASE_OPTS[@]}" \ - "root@$ip" 2>/dev/null + "$(_ssh_host "$ip")" 2>/dev/null local i for i in $(seq 1 30); do - if ssh -O check -o ControlPath="$socket" "root@$ip" 2>/dev/null; then + if ssh -O check -o ControlPath="$socket" "$(_ssh_host "$ip")" 2>/dev/null; then return 0 fi [ "$i" = "1" ] && echo "==> Establishing SSH connection to $ip..." @@ -57,7 +64,7 @@ ssh_close() { local ip="$1" local socket socket=$(_ssh_socket_path "$ip") - ssh -O exit -o ControlPath="$socket" "root@$ip" 2>/dev/null || true + ssh -O exit -o ControlPath="$socket" "$(_ssh_host "$ip")" 2>/dev/null || true rm -f "$socket" } @@ -74,7 +81,7 @@ ssh_run() { _ssh_open "$ip" || return 1 - ssh -T -o ControlPath="$socket" "${SSH_BASE_OPTS[@]}" "root@$ip" \ + ssh -T -o ControlPath="$socket" "${SSH_BASE_OPTS[@]}" "$(_ssh_host "$ip")" \ "echo $marker; $*; echo $marker" 2>/dev/null /dev/null /dev/null Running health checks..." set +e - ssh_once "$ip" \ + ssh -T "${SSH_BASE_OPTS[@]}" "$(_ssh_host "$ip")" \ "systemctl is-active s3-server.service && docker ps --format '{{.Names}} {{.Status}}'" HEALTH_EXIT=$? set -e diff --git a/docs/platform-support.md b/docs/platform-support.md index b212429..5575f07 100644 --- a/docs/platform-support.md +++ b/docs/platform-support.md @@ -12,31 +12,21 @@ Deployment targets for s3-server. All use the same Docker image (`ghcr.io/lumewe | Platform | Install | Update | Difficulty | Docs | |---|---|---|---|---| -| DigitalOcean | 1-Click Droplet or Packer snapshot | Sidecar (auto-update ON by default) | Easy | [README](../deploy/digitalocean/README.md) | -| Vultr | Marketplace App or Packer snapshot | Sidecar (auto-update ON by default) | Easy | [README](../deploy/vultr/README.md) | +| DigitalOcean 1-Click | Droplet from Marketplace image | Auto-update sidecar | Medium | [deploy/digitalocean/README.md](../deploy/digitalocean/README.md) | +| Vultr 1-Click | VPS from Marketplace image | Auto-update sidecar | Medium | [deploy/vultr/README.md](../deploy/vultr/README.md) | +| AWS EC2 / CloudFormation | EC2 from Marketplace AMI | Auto-update sidecar | Medium | [deploy/aws/README.md](../deploy/aws/README.md) | -## PaaS +## Platform Feature Parity -_Platform configs will be added here as providers are onboarded._ - -## Kubernetes - -_Platform configs will be added here as providers are onboarded._ - -## Update Mechanism Summary - -| Environment | Update Model | Panel UI | Sidecar | -|---|---|---|---| -| Docker Compose | Sidecar in compose | Auto-update toggle + manual "update now" button | Yes | -| Cloud VMs | Sidecar (Packer pre-installed) | Auto-update toggle + manual "update now" button | Yes | -| PaaS | Platform-native | None (platform manages updates) | No | -| Kubernetes | Manual helm upgrade / redeploy | Read-only status badge (no toggle/button) | No | +| Feature | Self-Hosted | DigitalOcean | Vultr | AWS | +|---|---|---|---|---| +| Docker Compose stack | ✅ | ✅ | ✅ | ✅ | +| Auto-update sidecar | Manual | ✅ | ✅ | ✅ | +| HTTPS/S3 domain configuration | Manual | Manual | Manual | Manual | +| CloudFormation deployment | ❌ | ❌ | ❌ | ✅ | +| Firewall configured | Manual | ✅ (ufw) | ✅ (ufw) | ✅ (Security Group + ufw) | -## Architecture +## Notes -- **Image**: `ghcr.io/lumeweb/s3-server:latest` (amd64 + arm64) -- **Updater image**: `ghcr.io/lumeweb/s3-server-updater:latest` (multi-arch, built by this repo's CI) -- **Binary**: `s3-server` (Go, SQLite, port 80) -- **Update sidecar**: polls GHCR `:latest` every 6h, compares digest, recreates container on change (~170 lines bash) -- **Sidecar flag files** (shared `/state` volume): `autoupdate.enabled`, `autoupdate.disabled`, `update.trigger`, `last-digest`, `updater.log` -- No `os/exec` in s3-server, no tini, no Docker-in-Docker, no Watchtower +- AWS requires the AMI to be published to AWS Marketplace and shared with the AWS Marketplace service account (`679593333241`) before buyers can deploy it. The CloudFormation template in `deploy/aws/cloudformation.template.json` references the AMI ID via an `AWS::EC2::Image::Id` parameter. +- All Marketplace images disable root login and password authentication. Administrative access is via the configured user (`root` on DigitalOcean, `root` on Vultr, or `ubuntu` with optional SSH on AWS). diff --git a/packer/scripts/install-s3-server.sh b/packer/scripts/install-s3-server.sh index e54ce64..2a49bf2 100755 --- a/packer/scripts/install-s3-server.sh +++ b/packer/scripts/install-s3-server.sh @@ -185,11 +185,11 @@ Pinner.xyz S3 Server - Private, zero-knowledge self-hosted S3-compatible object Auto-update is **enabled by default**. The update sidecar checks for new container images every 6 hours. To disable: ```bash -docker exec s3-deployment-updater-1 touch /state/autoupdate.disabled +touch /state/autoupdate.disabled ``` Re-enable with: ```bash -docker exec s3-deployment-updater-1 rm /state/autoupdate.disabled +rm /state/autoupdate.disabled ``` No restart needed. diff --git a/scripts/validate-aws.sh b/scripts/validate-aws.sh new file mode 100755 index 0000000..b922cfb --- /dev/null +++ b/scripts/validate-aws.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# scripts/validate-aws.sh: Local static validation for the AWS target. +# Does NOT require AWS credentials. +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +AWS_DIR="$ROOT/deploy/aws" +FAILED=0 + +echo "==> Validating AWS target artifacts..." + +# --- Packer --- +if command -v packer >/dev/null 2>&1; then + echo "--> Packer fmt check" + if ! packer fmt -check "$AWS_DIR/"; then + echo " FAIL: packer fmt check failed" + FAILED=1 + else + echo " PASS: packer fmt check" + fi + + echo "--> Packer init + validate" + if (cd "$AWS_DIR" && packer init . && packer validate -syntax-only .); then + echo " PASS: packer validate" + else + echo " FAIL: packer validate failed" + FAILED=1 + fi +else + echo "WARN: packer not found, skipping HCL validation" +fi + +# --- CloudFormation JSON --- +echo "--> CloudFormation JSON parse" +if python3 -c "import json; json.load(open('$AWS_DIR/cloudformation.template.json'))"; then + echo " PASS: CloudFormation JSON parses" +else + echo " FAIL: CloudFormation JSON parse error" + FAILED=1 +fi + +# --- Shell scripts --- +for SH in "$AWS_DIR/scripts/001_provision.sh" "$AWS_DIR/scripts/900-cleanup.sh" "$AWS_DIR/scripts/validate.sh"; do + echo "--> bash -n: $SH" + if bash -n "$SH"; then + echo " PASS: $SH" + else + echo " FAIL: $SH" + FAILED=1 + fi +done + +# --- Python scripts --- +for PY in "$AWS_DIR/submit.py" "$AWS_DIR/scripts/prune-snapshots.py" "$AWS_DIR/scripts/prune-instances.py"; do + echo "--> py_compile: $PY" + if python3 -m py_compile "$PY"; then + echo " PASS: $PY" + else + echo " FAIL: $PY" + FAILED=1 + fi +done + +# --- Makefile --- +echo "--> Makefile syntax check" +if make -C "$AWS_DIR" -n all >/dev/null 2>&1; then + echo " PASS: Makefile syntax" +else + echo " FAIL: Makefile syntax" + FAILED=1 +fi + +# --- Markdown frontmatter sanity --- +for MD in "$AWS_DIR/marketplace/USAGE_INSTRUCTIONS.md" "$AWS_DIR/marketplace/SECURITY_GROUPS.md" "$AWS_DIR/marketplace/SUBMISSION_CHECKLIST.md"; do + echo "--> file exists and non-empty: $MD" + if [ -s "$MD" ]; then + echo " PASS: $MD" + else + echo " FAIL: $MD missing or empty" + FAILED=1 + fi +done + +if [ "$FAILED" -ne 0 ]; then + echo "" + echo "Error: AWS validation failed" >&2 + exit 1 +fi + +echo "" +echo "==> AWS target validation passed"