From 04eb8c730fca3c1cfe5a5b1db45f13b91bd70ba1 Mon Sep 17 00:00:00 2001 From: Producdevity Date: Mon, 12 Oct 2026 00:10:53 +0200 Subject: [PATCH 1/3] fix(ci): build staging images with isolated public settings --- .github/workflows/container-image.yml | 19 +++++++++++------- docs/container-image.md | 28 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 docs/container-image.md diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index 4483d073..bb69da2c 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -18,10 +18,10 @@ concurrency: jobs: image: - if: github.ref == 'refs/heads/master' + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/staging' runs-on: ubuntu-latest timeout-minutes: 45 - environment: container-production + environment: ${{ github.ref == 'refs/heads/master' && 'container-production' || 'container-staging' }} env: DATABASE_URL: postgresql://postgres:postgres@127.0.0.1:5432/emuready_build DATABASE_DIRECT_URL: postgresql://postgres:postgres@127.0.0.1:5432/emuready_build @@ -69,10 +69,15 @@ jobs: raise SystemExit('Only public arguments declared in Dockerfile are accepted') if any(not isinstance(value, str) or any(c in value for c in '\r\n\0') for value in config.values()): raise SystemExit('Build values must be single-line strings') - if config.get('NEXT_PUBLIC_APP_URL') != 'https://www.emuready.com' or config.get('NEXT_PUBLIC_APP_ENV') != 'production': - raise SystemExit('Expected final production URL and environment') - if not config.get('NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY', '').startswith('pk_live_'): - raise SystemExit('Expected the production Clerk publishable key') + expected = { + 'refs/heads/master': ('https://www.emuready.com', 'production', 'pk_live_'), + 'refs/heads/staging': ('https://staging.emuready.com', 'staging', 'pk_test_'), + } + url, environment, clerk_prefix = expected[os.environ['GITHUB_REF']] + if config.get('NEXT_PUBLIC_APP_URL') != url or config.get('NEXT_PUBLIC_APP_ENV') != environment: + raise SystemExit('Public URL and environment must match the selected branch') + if not config.get('NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY', '').startswith(clerk_prefix): + raise SystemExit('Clerk publishable key must match the selected environment') args = [item for key, value in sorted(config.items()) for item in ('--build-arg', f'{key}={value}')] pathlib.Path(os.environ['RUNNER_TEMP'], 'container-build-args').write_bytes(('\0'.join(args) + '\0').encode()) PY @@ -88,7 +93,7 @@ jobs: - name: Run unit tests run: pnpm test:ci - - name: Build production image + - name: Build application image id: image run: | image="ghcr.io/${GITHUB_REPOSITORY,,}" diff --git a/docs/container-image.md b/docs/container-image.md new file mode 100644 index 00000000..b7301485 --- /dev/null +++ b/docs/container-image.md @@ -0,0 +1,28 @@ +# Container images + +The manual **Container image** workflow builds on GitHub runners and optionally +publishes to private GHCR. It does not deploy or change DNS. + +Configure `CONTAINER_PUBLIC_BUILD_ENV` as a JSON object in each GitHub environment: + +| Branch | GitHub environment | Public URL | App environment | Clerk key | +| --- | --- | --- | --- | --- | +| `master` | `container-production` | `https://www.emuready.com` | `production` | `pk_live_` | +| `staging` | `container-staging` | `https://staging.emuready.com` | `staging` | `pk_test_` | + +Restrict each environment to its corresponding branch. Only public Dockerfile +arguments are accepted. Never supply hosted database credentials or application +secrets: the workflow migrates an empty PostgreSQL 15 database on the runner for +Prisma generation. Public settings are baked into the image and require a rebuild +when changed. + +After the exact commit passes CI, dispatch the workflow on the corresponding +branch with `publish` enabled. Deploy the returned immutable digest through +Coolify's Docker Image build pack. Preserve runtime credentials, domains, labels, +port 3000, health checks, certificate mounts and persistent storage. Disable +source auto-deployment so pushes cannot start builds on the application VPS. +Set runtime `APP_VERSION` to the image's exact source commit. + +Verify image identity, readiness, TLS, canonical URLs, authentication and static +assets before switching public DNS. Keep the existing Vercel deployments and +aliases intact for rollback until the user explicitly approves retirement. From 2cda154cdbc2d707cecc45c48e0244ca181fcfb7 Mon Sep 17 00:00:00 2001 From: Producdevity Date: Mon, 12 Oct 2026 01:13:23 +0200 Subject: [PATCH 2/3] fix(ci): validate container settings in CI and document migration gates --- .github/workflows/ci.yml | 3 ++ .github/workflows/container-image.yml | 26 +---------- docs/SELF_HOSTING.md | 5 +- docs/container-image.md | 25 +++++++--- scripts/container_build_config.py | 36 ++++++++++++++ scripts/container_build_config_test.py | 65 ++++++++++++++++++++++++++ 6 files changed, 127 insertions(+), 33 deletions(-) create mode 100644 scripts/container_build_config.py create mode 100644 scripts/container_build_config_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8edb161..93bd0978 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,9 @@ jobs: - name: Checkout code uses: actions/checkout@v6 + - name: Test container build configuration + run: python3 -m unittest discover -s scripts -p container_build_config_test.py + - name: Setup pnpm uses: pnpm/action-setup@v6 diff --git a/.github/workflows/container-image.yml b/.github/workflows/container-image.yml index bb69da2c..d15ff05a 100644 --- a/.github/workflows/container-image.yml +++ b/.github/workflows/container-image.yml @@ -56,31 +56,7 @@ jobs: - name: Validate public build configuration env: PUBLIC_BUILD_ENV: ${{ vars.CONTAINER_PUBLIC_BUILD_ENV }} - run: | - python3 - <<'PY' - import json, os, pathlib, re - - config = json.loads(os.environ['PUBLIC_BUILD_ENV']) - if not isinstance(config, dict): - raise SystemExit('CONTAINER_PUBLIC_BUILD_ENV must be a JSON object') - declared = set(re.findall(r'^ARG ([A-Z0-9_]+)', pathlib.Path('Dockerfile').read_text(), re.M)) - allowed = {key for key in declared if key.startswith('NEXT_PUBLIC_') or key == 'NEXT_IMAGE_UNOPTIMIZED'} - if set(config) - allowed: - raise SystemExit('Only public arguments declared in Dockerfile are accepted') - if any(not isinstance(value, str) or any(c in value for c in '\r\n\0') for value in config.values()): - raise SystemExit('Build values must be single-line strings') - expected = { - 'refs/heads/master': ('https://www.emuready.com', 'production', 'pk_live_'), - 'refs/heads/staging': ('https://staging.emuready.com', 'staging', 'pk_test_'), - } - url, environment, clerk_prefix = expected[os.environ['GITHUB_REF']] - if config.get('NEXT_PUBLIC_APP_URL') != url or config.get('NEXT_PUBLIC_APP_ENV') != environment: - raise SystemExit('Public URL and environment must match the selected branch') - if not config.get('NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY', '').startswith(clerk_prefix): - raise SystemExit('Clerk publishable key must match the selected environment') - args = [item for key, value in sorted(config.items()) for item in ('--build-arg', f'{key}={value}')] - pathlib.Path(os.environ['RUNNER_TEMP'], 'container-build-args').write_bytes(('\0'.join(args) + '\0').encode()) - PY + run: python3 scripts/container_build_config.py - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/docs/SELF_HOSTING.md b/docs/SELF_HOSTING.md index 979c1560..3e0caeb8 100644 --- a/docs/SELF_HOSTING.md +++ b/docs/SELF_HOSTING.md @@ -12,7 +12,10 @@ EmuReady runs as a standalone Next.js container behind Coolify and Cloudflare. S - Keep the real `DATABASE_URL`, `DATABASE_DIRECT_URL`, and other runtime-only secrets, such as `CLERK_SECRET_KEY`, out of the build phase. Backups and environment files must remain excluded from the Docker context. - For a release containing migrations, build the `migrator` target from the same commit and run it with `DATABASE_DIRECT_URL` before deploying the `app` image. -The VPS currently builds from source in Coolify. Staging uses the GitHub App webhook to deploy its configured branch; production is configured for manual deployments. Publishing prebuilt immutable images remains deferred. +Use the manual [Container image workflow](container-image.md) to build and publish +images on GitHub runners, then deploy their immutable digests through Coolify's +Docker Image build pack. Disable source auto-deployment to keep builds off the +application VPS. Image publication does not deploy or apply hosted migrations. ## Coolify application diff --git a/docs/container-image.md b/docs/container-image.md index b7301485..e8a77874 100644 --- a/docs/container-image.md +++ b/docs/container-image.md @@ -3,23 +3,34 @@ The manual **Container image** workflow builds on GitHub runners and optionally publishes to private GHCR. It does not deploy or change DNS. -Configure `CONTAINER_PUBLIC_BUILD_ENV` as a JSON object in each GitHub environment: +Create a GitHub Actions environment **variable** named `CONTAINER_PUBLIC_BUILD_ENV` +containing a JSON object in each environment below. The workflow reads it through +`vars`; configuring an environment secret instead will leave this value empty. -| Branch | GitHub environment | Public URL | App environment | Clerk key | +| Branch | GitHub environment | Public URL | App environment | Clerk key prefix | | --- | --- | --- | --- | --- | | `master` | `container-production` | `https://www.emuready.com` | `production` | `pk_live_` | | `staging` | `container-staging` | `https://staging.emuready.com` | `staging` | `pk_test_` | Restrict each environment to its corresponding branch. Only public Dockerfile -arguments are accepted. Never supply hosted database credentials or application -secrets: the workflow migrates an empty PostgreSQL 15 database on the runner for +arguments are accepted. Supply the full Clerk publishable key in the JSON; the +table shows only the required prefix. Never supply hosted database credentials or +application secrets: the workflow migrates an empty PostgreSQL 15 database on the runner for Prisma generation. Public settings are baked into the image and require a rebuild when changed. After the exact commit passes CI, dispatch the workflow on the corresponding -branch with `publish` enabled. Deploy the returned immutable digest through -Coolify's Docker Image build pack. Preserve runtime credentials, domains, labels, -port 3000, health checks, certificate mounts and persistent storage. Disable +branch with `publish` enabled. Before deployment, check the target database's +applied migrations against that commit. This workflow migrates only its disposable +database and publishes only the `app` target; the application does not migrate on +startup. If migrations are pending, stop until a fresh backup is verified and the +user explicitly approves applying them. Build the same-commit `migrator` target +off the application VPS and apply the migrations as described in +[Self-hosting](SELF_HOSTING.md#build-and-release-contract) before deploying the app. + +Deploy the returned immutable digest through Coolify's Docker Image build pack. +Preserve runtime credentials, domains, labels, port 3000, health checks, +certificate mounts and persistent storage. Disable source auto-deployment so pushes cannot start builds on the application VPS. Set runtime `APP_VERSION` to the image's exact source commit. diff --git a/scripts/container_build_config.py b/scripts/container_build_config.py new file mode 100644 index 00000000..759e268a --- /dev/null +++ b/scripts/container_build_config.py @@ -0,0 +1,36 @@ +import json +import os +from pathlib import Path +import re + + +def build_arguments(config, ref, dockerfile): + if not isinstance(config, dict): + raise ValueError('CONTAINER_PUBLIC_BUILD_ENV must be a JSON object') + declared = set(re.findall(r'^ARG ([A-Z0-9_]+)', dockerfile, re.M)) + allowed = {key for key in declared if key.startswith('NEXT_PUBLIC_') or key == 'NEXT_IMAGE_UNOPTIMIZED'} + if set(config) - allowed: + raise ValueError('Only public arguments declared in Dockerfile are accepted') + if any(not isinstance(value, str) or any(c in value for c in '\r\n\0') for value in config.values()): + raise ValueError('Build values must be single-line strings') + expected = { + 'refs/heads/master': ('https://www.emuready.com', 'production', 'pk_live_'), + 'refs/heads/staging': ('https://staging.emuready.com', 'staging', 'pk_test_'), + } + if ref not in expected: + raise ValueError('Container images may only be built from master or staging') + url, environment, clerk_prefix = expected[ref] + if config.get('NEXT_PUBLIC_APP_URL') != url or config.get('NEXT_PUBLIC_APP_ENV') != environment: + raise ValueError('Public URL and environment must match the selected branch') + clerk_key = config.get('NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY', '') + if not clerk_key.startswith(clerk_prefix) or len(clerk_key) == len(clerk_prefix): + raise ValueError('Supply a full Clerk publishable key for the selected environment') + return [item for key, value in sorted(config.items()) for item in ('--build-arg', f'{key}={value}')] + + +if __name__ == '__main__': + try: + arguments = build_arguments(json.loads(os.environ['PUBLIC_BUILD_ENV']), os.environ['GITHUB_REF'], Path('Dockerfile').read_text()) + except ValueError as error: + raise SystemExit(str(error)) from error + Path(os.environ['RUNNER_TEMP'], 'container-build-args').write_bytes(('\0'.join(arguments) + '\0').encode()) diff --git a/scripts/container_build_config_test.py b/scripts/container_build_config_test.py new file mode 100644 index 00000000..cd8f8281 --- /dev/null +++ b/scripts/container_build_config_test.py @@ -0,0 +1,65 @@ +from pathlib import Path +import unittest + +from container_build_config import build_arguments + + +DOCKERFILE = (Path(__file__).resolve().parent.parent / 'Dockerfile').read_text() +STAGING = { + 'NEXT_PUBLIC_APP_URL': 'https://staging.emuready.com', + 'NEXT_PUBLIC_APP_ENV': 'staging', + 'NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY': 'pk_test_c3RhZ2luZy5leGFtcGxlLmNvbSQ=', +} +PRODUCTION = { + 'NEXT_PUBLIC_APP_URL': 'https://www.emuready.com', + 'NEXT_PUBLIC_APP_ENV': 'production', + 'NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY': 'pk_live_cHJvZHVjdGlvbi5leGFtcGxlLmNvbSQ=', +} + + +class ContainerBuildConfigTest(unittest.TestCase): + def test_accepts_each_branch_configuration(self): + for branch, config in [('master', PRODUCTION), ('staging', STAGING)]: + with self.subTest(branch=branch): + args = build_arguments(config, 'refs/heads/' + branch, DOCKERFILE) + self.assertIn('NEXT_PUBLIC_APP_URL=' + config['NEXT_PUBLIC_APP_URL'], args) + self.assertEqual(args.count('--build-arg'), len(config)) + + def test_rejects_individual_cross_environment_values(self): + for branch, expected, other in [('master', PRODUCTION, STAGING), ('staging', STAGING, PRODUCTION)]: + for key in expected: + with self.subTest(branch=branch, key=key): + with self.assertRaises(ValueError): + build_arguments({**expected, key: other[key]}, 'refs/heads/' + branch, DOCKERFILE) + + def test_rejects_missing_or_prefix_only_key(self): + for key in ('', 'pk_test_'): + with self.subTest(key=key): + with self.assertRaises(ValueError): + build_arguments({**STAGING, 'NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY': key}, 'refs/heads/staging', DOCKERFILE) + + def test_rejects_secret_and_undeclared_arguments(self): + for key in ('DATABASE_URL', 'BUILD_DATABASE_URL', 'CLERK_SECRET_KEY', 'NEXT_PUBLIC_UNKNOWN'): + with self.subTest(key=key): + with self.assertRaises(ValueError): + build_arguments({**STAGING, key: 'value'}, 'refs/heads/staging', DOCKERFILE) + + def test_rejects_non_object_and_unsafe_values(self): + for config in ([], None, {**STAGING, 'NEXT_PUBLIC_APP_URL': False}, + *({**STAGING, 'NEXT_PUBLIC_APP_URL': STAGING['NEXT_PUBLIC_APP_URL'] + char} for char in '\r\n\0')): + with self.subTest(config=config): + with self.assertRaises(ValueError): + build_arguments(config, 'refs/heads/staging', DOCKERFILE) + + def test_rejects_other_branches(self): + with self.assertRaises(ValueError): + build_arguments(STAGING, 'refs/heads/feature', DOCKERFILE) + + def test_keeps_shell_metacharacters_in_a_single_argument(self): + value = 'spaces $(echo test); *' + args = build_arguments({**STAGING, 'NEXT_PUBLIC_DISCORD_LINK': value}, 'refs/heads/staging', DOCKERFILE) + self.assertIn('NEXT_PUBLIC_DISCORD_LINK=' + value, args) + + +if __name__ == '__main__': + unittest.main() From db7d44d715e8a17c6d1a432d9968a4bcbd343c9c Mon Sep 17 00:00:00 2001 From: Producdevity Date: Mon, 12 Oct 2026 01:27:26 +0200 Subject: [PATCH 3/3] fix(ci): tighten public config checks and complete image runbook --- docs/SELF_HOSTING.md | 6 ++---- docs/container-image.md | 8 +++++++- scripts/container_build_config.py | 4 ++-- scripts/container_build_config_test.py | 9 ++++++++- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/docs/SELF_HOSTING.md b/docs/SELF_HOSTING.md index 3e0caeb8..df5366e8 100644 --- a/docs/SELF_HOSTING.md +++ b/docs/SELF_HOSTING.md @@ -8,7 +8,6 @@ EmuReady runs as a standalone Next.js container behind Coolify and Cloudflare. S - Supply all `NEXT_PUBLIC_*` values while building. Runtime values cannot change the browser bundle. - Set `BUILD_DATABASE_URL` to a migrated, disposable Postgres database because Prisma TypedSQL generation introspects the schema. Never use production or a restored production backup for this. Apply the existing migrations to an empty database; seeds are unnecessary. - Optionally set `BUILD_DATABASE_DIRECT_URL` for that same disposable database. It defaults to `BUILD_DATABASE_URL`. The builder maps these names to Prisma's `DATABASE_URL` and `DATABASE_DIRECT_URL` only for `pnpm build`, and fails if `BUILD_DATABASE_URL` is missing. -- In Coolify, make `BUILD_DATABASE_URL` and `BUILD_DATABASE_DIRECT_URL` build-only variables and enable **Use Docker Build Secrets**. Ordinary Docker build arguments expose their values in image metadata. - Keep the real `DATABASE_URL`, `DATABASE_DIRECT_URL`, and other runtime-only secrets, such as `CLERK_SECRET_KEY`, out of the build phase. Backups and environment files must remain excluded from the Docker context. - For a release containing migrations, build the `migrator` target from the same commit and run it with `DATABASE_DIRECT_URL` before deploying the `app` image. @@ -19,8 +18,8 @@ application VPS. Image publication does not deploy or apply hosted migrations. ## Coolify application -- Use the Dockerfile build pack, target `app`, and exposed port `3000`. -- Set `NEXT_BUILD_ID=$SOURCE_COMMIT` and enable **Include Source Commit in Build**. +- Use the Docker Image build pack with the verified immutable digest and exposed port `3000`. +- Set runtime `APP_VERSION` to the exact source commit. The image workflow sets `NEXT_BUILD_ID` while building. - Use `/api/health/ready` for deployment health checks and `/api/health/live` for process liveness. ## Production configuration @@ -33,7 +32,6 @@ application VPS. Image publication does not deploy or apply hosted migrations. - Provision an isolated staging upload bucket, scoped token, and hostname before enabling upload testing in staging. - Make APK objects private so entitlement checks cannot be bypassed with a known public R2 URL. -- Move builds to GitHub-hosted Actions, publish immutable images to GHCR, and have Coolify deploy them by digest. Do not run the build runner on the application VPS. - Replace the current `staging` default branch and `master` production convention with a documented release and promotion flow. - Consolidate the duplicate mobile tRPC paths and remove the unused transport. - Define how authored handheld and PC Compatibility Reports are retained when a Clerk user is deleted; their required author relations currently block deletion (observed as `Listing_authorId_fkey`). diff --git a/docs/container-image.md b/docs/container-image.md index e8a77874..f450fda2 100644 --- a/docs/container-image.md +++ b/docs/container-image.md @@ -28,7 +28,13 @@ user explicitly approves applying them. Build the same-commit `migrator` target off the application VPS and apply the migrations as described in [Self-hosting](SELF_HOSTING.md#build-and-release-contract) before deploying the app. -Deploy the returned immutable digest through Coolify's Docker Image build pack. +Before deployment, configure GHCR authentication for the Docker account used by +Coolify on every deployment server. Use a pull-only credential with `read:packages` +and access to the private EmuReady package; track its expiration and rotate it +before it expires. Verify that the server can pull the exact published digest. +The workflow's GitHub token authenticates only the publishing runner. + +Deploy that immutable digest through Coolify's Docker Image build pack. Preserve runtime credentials, domains, labels, port 3000, health checks, certificate mounts and persistent storage. Disable source auto-deployment so pushes cannot start builds on the application VPS. diff --git a/scripts/container_build_config.py b/scripts/container_build_config.py index 759e268a..38c7bcb6 100644 --- a/scripts/container_build_config.py +++ b/scripts/container_build_config.py @@ -11,8 +11,8 @@ def build_arguments(config, ref, dockerfile): allowed = {key for key in declared if key.startswith('NEXT_PUBLIC_') or key == 'NEXT_IMAGE_UNOPTIMIZED'} if set(config) - allowed: raise ValueError('Only public arguments declared in Dockerfile are accepted') - if any(not isinstance(value, str) or any(c in value for c in '\r\n\0') for value in config.values()): - raise ValueError('Build values must be single-line strings') + if any(not isinstance(value, str) or value != value.strip() or any(c in value for c in '\r\n\0') for value in config.values()): + raise ValueError('Build values must be trimmed single-line strings') expected = { 'refs/heads/master': ('https://www.emuready.com', 'production', 'pk_live_'), 'refs/heads/staging': ('https://staging.emuready.com', 'staging', 'pk_test_'), diff --git a/scripts/container_build_config_test.py b/scripts/container_build_config_test.py index cd8f8281..61e5aed4 100644 --- a/scripts/container_build_config_test.py +++ b/scripts/container_build_config_test.py @@ -22,7 +22,8 @@ def test_accepts_each_branch_configuration(self): for branch, config in [('master', PRODUCTION), ('staging', STAGING)]: with self.subTest(branch=branch): args = build_arguments(config, 'refs/heads/' + branch, DOCKERFILE) - self.assertIn('NEXT_PUBLIC_APP_URL=' + config['NEXT_PUBLIC_APP_URL'], args) + self.assertEqual(set(args[1::2]), {f'{key}={value}' for key, value in config.items()}) + self.assertEqual(len(args), 2 * len(config)) self.assertEqual(args.count('--build-arg'), len(config)) def test_rejects_individual_cross_environment_values(self): @@ -44,6 +45,12 @@ def test_rejects_secret_and_undeclared_arguments(self): with self.assertRaises(ValueError): build_arguments({**STAGING, key: 'value'}, 'refs/heads/staging', DOCKERFILE) + def test_rejects_surrounding_whitespace_in_clerk_key(self): + for prefix, suffix in [(' ', ''), ('', ' '), ('\t', ''), ('', '\t')]: + with self.subTest(prefix=prefix, suffix=suffix): + with self.assertRaises(ValueError): + build_arguments({**STAGING, 'NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY': prefix + STAGING['NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY'] + suffix}, 'refs/heads/staging', DOCKERFILE) + def test_rejects_non_object_and_unsafe_values(self): for config in ([], None, {**STAGING, 'NEXT_PUBLIC_APP_URL': False}, *({**STAGING, 'NEXT_PUBLIC_APP_URL': STAGING['NEXT_PUBLIC_APP_URL'] + char} for char in '\r\n\0')):