From 6ebe8a30d6674ecc9c194869ec55436f776302b6 Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Sun, 19 Jul 2026 16:00:33 +0200 Subject: [PATCH 01/13] Work towards generic deploys --- .env.example | 9 +++ README.md | 41 +++++++++++++- docker-compose.yml | 18 ++++++ scripts/data.sh | 136 ++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 188 insertions(+), 16 deletions(-) create mode 100644 docker-compose.yml diff --git a/.env.example b/.env.example index 3fb730d5..fe7c6e6d 100644 --- a/.env.example +++ b/.env.example @@ -6,3 +6,12 @@ ADMIN_PASSWORD="your-secure-password-here" # AWS_REGION="auto" # AWS_ACCESS_KEY_ID="" # AWS_SECRET_ACCESS_KEY="" + +# Generic VPS deployment (optional) — targets the npm run data:* commands at +# a host reachable over plain ssh instead of Fly.io. The values below match +# the docker-compose setup from README → Deploy to a VPS; only DEPLOY_HOST +# and HOST_DATA_DIR need adjusting to your server. +# DEPLOY_HOST="deploy@my-vps.example.com" +# RESTART_CMD="docker restart editable" +# REMOTE_EXEC="docker exec editable" +# HOST_DATA_DIR="/srv/my-site/data" diff --git a/README.md b/README.md index 6e403ff1..6b697fa7 100644 --- a/README.md +++ b/README.md @@ -636,7 +636,7 @@ From here it's just iteration: add calls to action, give editors control over th Deploy your local site to a public URL in a few steps. -Editable runs on any VPS — all you need is Node.js, and the included `Dockerfile` works with any platform that supports Docker. The repository ships ready-made for [Fly.io](https://fly.io): install [flyctl](https://fly.io/docs/flyctl/install/), then sign in (opens your browser; creates a free account if you don't have one): +Editable runs on any VPS — all you need is Node.js, and the included `Dockerfile` works with any platform that supports Docker (see [Deploy to a VPS](#deploy-to-a-vps-experimental)). The repository ships ready-made for [Fly.io](https://fly.io), which remains the recommended default: machines scale to zero and wake in well under a second, so a personal site costs next to nothing to run. Install [flyctl](https://fly.io/docs/flyctl/install/), then sign in (opens your browser; creates a free account if you don't have one): ```sh fly auth login @@ -692,11 +692,48 @@ fly open Because each checkout manages exactly one app (see [Your site is your repo](#your-site-is-your-repo)), the target always comes from `fly.toml` — there's no app name to get wrong. If you ever do need to address a different app (say, a staging copy), every `fly` command and data script accepts `-a ` as an explicit override. +### Deploy to a VPS (experimental) + +The same image runs on any host with Docker — a DigitalOcean droplet, a Hetzner or Nodion VPS, a home server. The included `docker-compose.yml` is the deployment; the data commands ([Backup, sync & recovery](#backup-sync--recovery)) and [automated backups](#automated-backups-optional) work identically, they just reach the server over plain ssh instead of the fly CLI. + +On the server (assuming Docker and ssh key access are set up), clone your site and configure it: + +```sh +git clone /srv/my-site && cd /srv/my-site +cp .env.example .env +# set in .env: ADMIN_PASSWORD, and ORIGIN="https://my-site.example.com" +# optional: the BUCKET_* / AWS_* secrets for automated backups +docker compose up -d --build +``` + +The app listens on `127.0.0.1:3000`; put a reverse proxy with TLS in front. With [Caddy](https://caddyserver.com) that's the whole config: + +``` +my-site.example.com { + reverse_proxy 127.0.0.1:3000 +} +``` + +To ship a code update: push to your repo, then `ssh` in and `git pull && docker compose up -d --build`. + +On your **local machine**, point the data commands at the server by uncommenting the deployment block in `.env`: + +```sh +DEPLOY_HOST="deploy@my-site.example.com" # who you ssh in as +RESTART_CMD="docker restart editable" +REMOTE_EXEC="docker exec editable" +HOST_DATA_DIR="/srv/my-site/data" # the ./data bind mount, as an absolute path +``` + +That's the entire switch: with `DEPLOY_HOST` set, every `npm run data:*` command targets the VPS (`fly.toml` is ignored); remove or comment it to target Fly.io again. Pull, push, backups, restores, point-in-time recovery — the whole toolbox behaves the same, including disaster recovery from the backup bucket on a fresh volume. + +What Fly.io still does for you that a VPS doesn't: scale-to-zero with sub-second wake-ups (a VPS runs — and bills — around the clock), TLS and anycast routing without a reverse proxy, and volume snapshots. The VPS path trades that for a fixed monthly price and no platform dependency. + ## Backup, sync & recovery Your whole site lives in one folder — pull it, push it, snapshot it, roll it back. -That folder is `data/`: an SQLite database (`db.sqlite3`) and uploaded assets (`assets/`). Locally it defaults to `./data`; on Fly.io it's a persistent volume at `/data`. The data commands move that folder between your machine, your deployment, and — optionally — a backup bucket. The complete toolbox: +That folder is `data/`: an SQLite database (`db.sqlite3`) and uploaded assets (`assets/`). Locally it defaults to `./data`; on Fly.io it's a persistent volume at `/data`; on a VPS it's the `./data` bind mount next to the compose file. The data commands move that folder between your machine, your deployment, and — optionally — a backup bucket. The complete toolbox: - **npm run data:pull** — Copy the live site's data to your machine - **npm run data:push [-- --yes]** — Replace the live site's data with your local state — guarded, undoable diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..a560afd7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +# Run Editable on any docker host — a DigitalOcean droplet, a Hetzner or +# Nodion VPS, a home server. See README → Deploy to a VPS. Fly.io deployments +# don't use this file (fly deploy builds the same Dockerfile directly). +# +# The app listens on 127.0.0.1:3000 — put a reverse proxy with TLS in front +# (Caddy makes that two lines, see the README). +services: + editable: + build: . + container_name: editable + restart: unless-stopped + ports: + - '127.0.0.1:3000:3000' + env_file: .env + environment: + BODY_SIZE_LIMIT: '${BODY_SIZE_LIMIT:-30000000}' + volumes: + - ./data:/data diff --git a/scripts/data.sh b/scripts/data.sh index 51ae57a8..ac80528d 100755 --- a/scripts/data.sh +++ b/scripts/data.sh @@ -1,7 +1,13 @@ #!/usr/bin/env bash # # Sync, back up, and recover the editable-website data folder (SQLite DB + -# content-addressed assets) between your local machine and a Fly.io deployment. +# content-addressed assets) between your local machine and a deployment. +# +# Deployments are reached through a driver: 'fly' (Fly.io, the default) or +# 'ssh' (any VPS reachable over plain ssh, running the app via docker compose +# or bare node). All provider-specific behavior funnels through five +# primitives — ensure_running, remote, sftp_get, sftp_put, restart_app — +# everything else is identical across providers. # # Safety model # - The database is copied only as a `VACUUM INTO` snapshot — never a raw file @@ -27,14 +33,25 @@ # ./scripts/data.sh reset # reset local database to fresh demo content # ./scripts/data.sh help # print the command reference # -# The target app is read from fly.toml (app = '...'), same as the fly CLI. -# Override with -a or the FLY_APP environment variable (-a wins). +# Fly driver: the target app is read from fly.toml (app = '...'), same as the +# fly CLI. Override with -a or the FLY_APP environment variable (-a wins). +# +# Ssh driver: configured via environment variables or .env (environment wins): +# DEPLOY_HOST user@host to ssh into (setting this selects the driver) +# RESTART_CMD how to restart the app, e.g. 'docker restart editable' +# REMOTE_EXEC command prefix to enter the app context, e.g. +# 'docker exec editable' (empty = bare node on the host) +# REMOTE_APP_DIR app dir inside the exec context (default /app) +# REMOTE_DATA_DIR data dir inside the exec context (default /data) +# HOST_DATA_DIR data dir as visible to scp on the host — set when it's a +# docker bind mount (default: REMOTE_DATA_DIR) +# DEPLOY_NAME label for backups and messages (default: the host) +# DEPLOY_DRIVER 'fly' or 'ssh' — set explicitly to override the inference # set -euo pipefail DATA_DIR_LOCAL="${DATA_DIR:-data}" BACKUP_DIR_LOCAL="data-backups" -REMOTE_DATA="/data" KEEP_BACKUPS="${KEEP_BACKUPS:-10}" APP="${FLY_APP:-}" ASSET_RE='^[a-f0-9]{64}(\.|$)' @@ -61,6 +78,41 @@ if [ -z "$APP" ] && [ -f "$SCRIPT_DIR/../fly.toml" ]; then APP="$(sed -n -E "s/^app[[:space:]]*=[[:space:]]*[\"']?([A-Za-z0-9-]+).*/\1/p" "$SCRIPT_DIR/../fly.toml" | sed -n '1p')" fi +# ---- deploy driver ----------------------------------------------------------- +# Ssh-driver configuration may live in .env — the environment wins over .env. +if [ -f "$SCRIPT_DIR/../.env" ]; then + while IFS='=' read -r key value; do + case "$key" in + DEPLOY_DRIVER | DEPLOY_HOST | DEPLOY_NAME | REMOTE_APP_DIR | REMOTE_DATA_DIR | HOST_DATA_DIR | REMOTE_EXEC | RESTART_CMD) + # Strip one layer of surrounding quotes. + value="${value%\"}"; value="${value#\"}" + value="${value%\'}"; value="${value#\'}" + eval "current=\${$key:-}" + [ -n "$current" ] || eval "$key=\$value" + ;; + esac + done <"$SCRIPT_DIR/../.env" +fi + +DRIVER="${DEPLOY_DRIVER:-}" +if [ -z "$DRIVER" ]; then + if [ -n "${DEPLOY_HOST:-}" ]; then DRIVER=ssh; else DRIVER=fly; fi +fi +[ "$DRIVER" = "fly" ] || [ "$DRIVER" = "ssh" ] || { + echo "Error: Unknown DEPLOY_DRIVER '$DRIVER' (expected 'fly' or 'ssh')" >&2 + exit 2 +} + +DEPLOY_HOST="${DEPLOY_HOST:-}" +REMOTE_APP_DIR="${REMOTE_APP_DIR:-/app}" +REMOTE_DATA="${REMOTE_DATA_DIR:-/data}" +HOST_DATA_DIR="${HOST_DATA_DIR:-$REMOTE_DATA}" +REMOTE_EXEC="${REMOTE_EXEC:-}" + +if [ "$DRIVER" = "ssh" ]; then + APP="${DEPLOY_NAME:-${DEPLOY_HOST#*@}}" +fi + TMP="$(mktemp -d)" trap 'rm -rf "$TMP"' EXIT @@ -71,7 +123,11 @@ info() { echo "→ $*"; } plural() { [ "$1" -eq 1 ] && echo "$1 $2" || echo "$1 ${3:-${2}s}"; } need_app() { - [ -n "$APP" ] || die "No app configured — set app = 'my-site' in fly.toml, or pass -a " + if [ "$DRIVER" = "ssh" ]; then + [ -n "${DEPLOY_HOST:-}" ] || die "No deploy host configured — set DEPLOY_HOST='user@host' in .env or the environment" + else + [ -n "$APP" ] || die "No app configured — set app = 'my-site' in fly.toml, or pass -a " + fi } confirm() { @@ -83,6 +139,12 @@ confirm() { MID="" ensure_running() { + if [ "$DRIVER" = "ssh" ]; then + # A VPS is always on — just confirm it's reachable before starting work. + ssh -o ConnectTimeout=10 "$DEPLOY_HOST" true >/dev/null 2>&1 || + die "Cannot reach '$DEPLOY_HOST' over ssh" + return + fi # sed (not head) reads all input, avoiding a SIGPIPE that pipefail would trip. # flyctl pads the -q output with whitespace, so strip it. MID="$(fly machine list -a "$APP" -q | sed -n '1p' | tr -d '[:space:]')" @@ -118,12 +180,53 @@ verify_remote() { printf '%s' "$out" | grep -q '^OK:' || die "Remote references missing assets — $ctx" } -# Run the in-container helper over SSH (pty-less, binary-safe). +# Run the in-app helper over SSH (pty-less, binary-safe). REMOTE_EXEC enters +# the app context on generic hosts (e.g. 'docker exec editable'); the env +# prefix runs inside that context, so DATA_DIR always names the in-app path. remote() { - fly ssh console -a "$APP" --machine "$MID" -C "sh /app/scripts/remote-db.sh $*" + if [ "$DRIVER" = "ssh" ]; then + # shellcheck disable=SC2029 + ssh "$DEPLOY_HOST" "${REMOTE_EXEC:+$REMOTE_EXEC }env DATA_DIR='$REMOTE_DATA' sh $REMOTE_APP_DIR/scripts/remote-db.sh $*" + else + fly ssh console -a "$APP" --machine "$MID" -C "sh $REMOTE_APP_DIR/scripts/remote-db.sh $*" + fi +} + +# Translate an in-app data path to the path scp sees on the host — they only +# differ when the data dir is a docker bind mount (HOST_DATA_DIR). +host_path() { + case "$1" in + "$REMOTE_DATA"/*) printf '%s%s' "$HOST_DATA_DIR" "${1#"$REMOTE_DATA"}" ;; + *) printf '%s' "$1" ;; + esac +} + +sftp_get() { + if [ "$DRIVER" = "ssh" ]; then + scp -q "$DEPLOY_HOST:$(host_path "$1")" "$2" + else + fly ssh sftp get -a "$APP" --machine "$MID" "$1" "$2" + fi +} +sftp_put() { + if [ "$DRIVER" = "ssh" ]; then + scp -q "$1" "$DEPLOY_HOST:$(host_path "$2")" + else + fly ssh sftp put -a "$APP" --machine "$MID" "$1" "$2" + fi +} + +# Restart the app so the boot-time promote swaps in a staged database. +restart_app() { + if [ "$DRIVER" = "ssh" ]; then + [ -n "${RESTART_CMD:-}" ] || + die "Set RESTART_CMD (e.g. 'docker restart editable' or 'systemctl restart editable') so the staged database can be swapped in" + # shellcheck disable=SC2029 + ssh "$DEPLOY_HOST" "$RESTART_CMD" + else + fly machine restart "$MID" -a "$APP" + fi } -sftp_get() { fly ssh sftp get -a "$APP" --machine "$MID" "$1" "$2"; } -sftp_put() { fly ssh sftp put -a "$APP" --machine "$MID" "$1" "$2"; } # Backup names double as identifiers, so they must be unique even for two # operations in the same second (a random suffix, not just seconds). They @@ -193,14 +296,18 @@ cmd_push() { remote promote-stage info "Restarting to swap in the new database…" - fly machine restart "$MID" -a "$APP" + restart_app info "Verifying…" verify_remote "restore with: npm run data:restore $ts" echo echo "✓ Pushed to '$APP'. Undo with:" - echo " npm run data:restore $ts -- -a $APP" + if [ "$DRIVER" = "fly" ]; then + echo " npm run data:restore $ts -- -a $APP" + else + echo " npm run data:restore $ts" + fi } # ---- pull: remote -> local ------------------------------------------------- @@ -307,7 +414,7 @@ cmd_restore() { fi info "Restarting to swap in the restored database…" - fly machine restart "$MID" -a "$APP" + restart_app info "Verifying…" local out @@ -369,7 +476,7 @@ cmd_restore_cloud() { remote promote-stage info "Restarting to swap in the restored database…" - fly machine restart "$MID" -a "$APP" + restart_app info "Verifying…" verify_remote "roll back with: npm run data:restore $ts" @@ -524,7 +631,8 @@ Data commands (via npm run; positional arguments work directly, flags need a -- npm run data:reset [-- --yes] reset local database to fresh demo content (assets stay) npm run litestream:install one-time local setup for the cloud commands -The target app comes from fly.toml; override with: -- -a +The target comes from fly.toml (Fly.io, override with: -- -a ) or from +DEPLOY_HOST in .env (any VPS over plain ssh — see README → Deploy to a VPS). Snapshot names () look like my-site-20260712T143535Z-3f2a (file extension optional) — list them with data:backups. Timestamps () are RFC3339 UTC, e.g. 2026-07-12T14:35:35Z — list them with data:cloud-snapshots. See README → Backup, sync & recovery for details. From 98b154e437ee87c015cf5d21fa93ba437bd127bd Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Mon, 20 Jul 2026 22:05:37 +0200 Subject: [PATCH 02/13] Add VPS deploy script --- .env.example | 12 +- README.md | 34 ++--- VPS_DEPLOY_SPEC.md | 98 ++++++++++++++ docker-compose.yml | 9 +- package.json | 1 + scripts/deploy_vps.sh | 308 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 435 insertions(+), 27 deletions(-) create mode 100644 VPS_DEPLOY_SPEC.md create mode 100755 scripts/deploy_vps.sh diff --git a/.env.example b/.env.example index fe7c6e6d..925f120e 100644 --- a/.env.example +++ b/.env.example @@ -8,10 +8,10 @@ ADMIN_PASSWORD="your-secure-password-here" # AWS_SECRET_ACCESS_KEY="" # Generic VPS deployment (optional) — targets the npm run data:* commands at -# a host reachable over plain ssh instead of Fly.io. The values below match -# the docker-compose setup from README → Deploy to a VPS; only DEPLOY_HOST -# and HOST_DATA_DIR need adjusting to your server. -# DEPLOY_HOST="deploy@my-vps.example.com" -# RESTART_CMD="docker restart editable" -# REMOTE_EXEC="docker exec editable" +# a host reachable over plain ssh instead of Fly.io. scripts/deploy_vps.sh +# prints this block with the right values after the first deploy; for a +# hand-managed compose setup the container is named plain "editable". +# DEPLOY_HOST="root@my-site.example.com" +# RESTART_CMD="docker restart editable-my-site" +# REMOTE_EXEC="docker exec editable-my-site" # HOST_DATA_DIR="/srv/my-site/data" diff --git a/README.md b/README.md index 6b697fa7..ac6cee50 100644 --- a/README.md +++ b/README.md @@ -694,38 +694,32 @@ Because each checkout manages exactly one app (see [Your site is your repo](#you ### Deploy to a VPS (experimental) -The same image runs on any host with Docker — a DigitalOcean droplet, a Hetzner or Nodion VPS, a home server. The included `docker-compose.yml` is the deployment; the data commands ([Backup, sync & recovery](#backup-sync--recovery)) and [automated backups](#automated-backups-optional) work identically, they just reach the server over plain ssh instead of the fly CLI. - -On the server (assuming Docker and ssh key access are set up), clone your site and configure it: +Editable runs on any amd64 host with Docker — a DigitalOcean droplet, a Hetzner or Nodion VPS. One command takes a fresh Ubuntu server to a running site with TLS. Create the server with your ssh key installed, point your domain's A record at its IP, then: ```sh -git clone /srv/my-site && cd /srv/my-site -cp .env.example .env -# set in .env: ADMIN_PASSWORD, and ORIGIN="https://my-site.example.com" -# optional: the BUCKET_* / AWS_* secrets for automated backups -docker compose up -d --build +npm run deploy:vps -- root@203.0.113.10 my-site.example.com ``` -The app listens on `127.0.0.1:3000`; put a reverse proxy with TLS in front. With [Caddy](https://caddyserver.com) that's the whole config: +The first run provisions the server — Docker, [Caddy](https://caddyserver.com) as the TLS-terminating reverse proxy, swap on machines with less than 2 GB RAM — asks you for an admin password, builds the Docker image locally, streams it over ssh, and starts the site. The server never needs access to your git repository or the memory to run a build. Every later run of the same command ships an update: build, stream, replace the container, health-check. Your content lives in `/srv/my-site/data` on the server; deploys replace the container and never touch that folder. -``` -my-site.example.com { - reverse_proxy 127.0.0.1:3000 -} -``` +Roll back a bad deploy by starting a previous image (the last three are kept on the server): -To ship a code update: push to your repo, then `ssh` in and `git pull && docker compose up -d --build`. +```sh +npm run deploy:vps -- root@203.0.113.10 my-site.example.com --tag +``` -On your **local machine**, point the data commands at the server by uncommenting the deployment block in `.env`: +After the first deploy the script prints the deployment block for your **local** `.env`, which points the data commands ([Backup, sync & recovery](#backup-sync--recovery)) at the server: ```sh -DEPLOY_HOST="deploy@my-site.example.com" # who you ssh in as -RESTART_CMD="docker restart editable" -REMOTE_EXEC="docker exec editable" +DEPLOY_HOST="root@203.0.113.10" # who you ssh in as +RESTART_CMD="docker restart editable-my-site" +REMOTE_EXEC="docker exec editable-my-site" HOST_DATA_DIR="/srv/my-site/data" # the ./data bind mount, as an absolute path ``` -That's the entire switch: with `DEPLOY_HOST` set, every `npm run data:*` command targets the VPS (`fly.toml` is ignored); remove or comment it to target Fly.io again. Pull, push, backups, restores, point-in-time recovery — the whole toolbox behaves the same, including disaster recovery from the backup bucket on a fresh volume. +That's the entire switch: with `DEPLOY_HOST` set, every `npm run data:*` command targets the VPS (`fly.toml` is ignored); remove or comment it to target Fly.io again. Pull, push, backups, restores, point-in-time recovery — the whole toolbox behaves the same, including disaster recovery from the backup bucket on a fresh volume. For [automated backups](#automated-backups-optional), the `BUCKET_*` / `AWS_*` secrets belong in the server's `.env` — the script copies them from your local `.env` on the first deploy, and `--push-env` re-syncs them later. + +**Doing it by hand instead:** the script is optional — `docker-compose.yml` runs on any docker host. Clone your site on the server, `cp .env.example .env` and set `ADMIN_PASSWORD` and `ORIGIN="https://my-site.example.com"`, then `docker compose up -d --build`. The app listens on `127.0.0.1:3000`; put a reverse proxy with TLS in front (with Caddy that's the whole config: `my-site.example.com { reverse_proxy 127.0.0.1:3000 }`). Ship updates with `git pull && docker compose up -d --build`, and use `RESTART_CMD="docker restart editable"` / `REMOTE_EXEC="docker exec editable"` in the deployment block above (without the script, the container keeps the default name `editable`). What Fly.io still does for you that a VPS doesn't: scale-to-zero with sub-second wake-ups (a VPS runs — and bills — around the clock), TLS and anycast routing without a reverse proxy, and volume snapshots. The VPS path trades that for a fixed monthly price and no platform dependency. diff --git a/VPS_DEPLOY_SPEC.md b/VPS_DEPLOY_SPEC.md new file mode 100644 index 00000000..f67600cf --- /dev/null +++ b/VPS_DEPLOY_SPEC.md @@ -0,0 +1,98 @@ +# VPS deploy script — specification + +Spec for `scripts/deploy_vps.sh`: a single local script that takes a fresh Ubuntu VPS (e.g. a DigitalOcean droplet) from zero to a running, TLS-terminated Editable site, and afterwards ships code updates to the same box. Modeled on the Writebook/ONCE installer experience, without Kamal or any registry. + +This file is the working spec for the feature. Once implemented and stable, fold the design decisions into `ARCHITECTURE.md` and the user-facing instructions into README → Deploy to a VPS. + +## Goals + +- One command against a fresh VPS sets up everything: `./scripts/deploy_vps.sh root@203.0.113.10 my-site.example.com` +- The same command run again ships a code update (the script detects what's needed; no separate setup/deploy modes for the user to learn) +- The image is built locally and streamed over ssh — the server never needs git access, npm, or the memory to run a Vite build +- The only state on the server that matters is the bind-mounted data directory; the container is disposable and replaced on every deploy +- A destroyed server is recovered by running the script against a fresh one (boot-time disaster recovery from the backup bucket already exists in `scripts/run-cloud-boot.js`) + +## Non-goals (v1) + +- Zero-downtime deploys — a few seconds of downtime while the container is replaced is accepted +- Multiple servers, multiple apps per server, deployment locking +- ARM servers — the Dockerfile pins the Litestream `.deb` to `linux-x86_64`, so v1 requires an amd64 VPS and builds with `--platform linux/amd64` (revisit by making the Litestream download arch-aware) +- Provisioning the VPS itself or DNS — the user creates the droplet (with their ssh key) and points the domain's A record at it first, like the Writebook flow + +## Design decisions + +**Build locally, stream over ssh, no registry.** `docker buildx build --platform linux/amd64` with the image tagged `editable:`, piped through gzip into `ssh 'gunzip | docker load'`. This keeps the server free of build tooling and repo access, and needs no registry account. + +**Host-level Caddy, not a Caddy container.** The app already binds `127.0.0.1:3000` and the README already documents the two-line Caddyfile. Caddy is installed from its apt repo and configured with exactly that config; certificates are Caddy's problem. No compose networking, no cert volumes. + +**Compose stays the runtime, switched from `build:` to `image:`.** `docker-compose.yml` gains `image: 'editable:${IMAGE_TAG:-local}'` and drops `build: .` as the on-server path. Local from-source runs use `docker compose up --build` explicitly (compose builds the `image:` tag when `--build` is passed), so the local workflow documented today keeps working. The script deploys with `IMAGE_TAG= docker compose up -d --remove-orphans`; compose sees the tag change and replaces the container, leaving the `./data` bind mount untouched. + +**Server layout.** Everything lives in one directory, `/srv/` (site name = the domain's first label, e.g. `my-site`): + +``` +/srv/my-site/ +├── docker-compose.yml uploaded by the script on every deploy +├── .env created on first run, never overwritten +└── data/ the persistent site data (bind mount → /data) +``` + +`/etc/caddy/Caddyfile` holds the reverse-proxy config. + +**Secrets.** On first run the script prompts for `ADMIN_PASSWORD` (offering a generated one), sets `ORIGIN=https://`, and writes the server's `.env` over sftp. Backup-bucket credentials (`BUCKET_NAME`, `AWS_*`) are copied from the local `.env` if present. On later runs the server `.env` is left alone; a `--push-env` flag re-uploads the bucket/origin values (never silently — it prints what changes). Secrets are never passed as command-line arguments to remote shells. + +**Idempotent provisioning, not a setup mode.** Every run executes the same phases; each phase checks before it acts (Docker installed? Caddy installed? Caddyfile current? `.env` present?). First run does everything; later runs fall through to the deploy phase in seconds. + +**Rollback = redeploy an old tag + existing data restore.** The script keeps the last 3 image tags on the server (older ones pruned after a successful deploy). `./scripts/deploy_vps.sh --tag ` starts that image instead of building. Content rollback is out of scope — that's `npm run data:restore`, which already works against the VPS via `DEPLOY_HOST`. + +**Health check gates success.** After `compose up`, the script polls `127.0.0.1:3000` over ssh (curl, ~30 s budget). On failure it prints the container logs and exits non-zero; it does not auto-rollback in v1. + +**`.env` bridge to the data toolbox.** After a successful first deploy the script prints the exact `DEPLOY_HOST` / `RESTART_CMD` / `REMOTE_EXEC` / `HOST_DATA_DIR` block for the local `.env` (values it already knows), so `npm run data:*` works immediately. + +## Script interface + +``` +./scripts/deploy_vps.sh [options] + +Options: + --tag deploy an already-uploaded image tag (rollback) instead of building + --push-env re-sync ORIGIN and bucket credentials into the server .env + --yes skip confirmation prompts (except the first-run password prompt) +``` + +`` is typically `root@` on a fresh droplet; any sudo-capable user works. ssh key access is assumed (the script never handles passwords). + +## Execution phases + +1. **Preflight (local).** Verify: git worktree present, `docker buildx` available, ssh connectivity to the host (`BatchMode=yes`), remote architecture is x86_64 (abort otherwise), domain resolves to the host's IP (warn, don't abort — DNS may still be propagating). +2. **Provision (remote, idempotent).** Install Docker (official convenience script) and Caddy (apt repo) if missing; create 1 GB swap if total RAM < 2 GB and no swap exists; create `/srv//data`. +3. **Configure (remote, idempotent).** Write `/etc/caddy/Caddyfile` (reverse_proxy block for the domain) and reload Caddy if it changed. First run: prompt for `ADMIN_PASSWORD`, write `.env` with it plus `ORIGIN` and any local bucket credentials. +4. **Build & upload (local → remote).** Skipped with `--tag`. Build `editable:` for linux/amd64, stream via `ssh docker load`. Upload `docker-compose.yml`. +5. **Activate (remote).** `IMAGE_TAG= docker compose up -d --remove-orphans` in `/srv/` (tag passed via a `.deploy_tag` env file, not shell interpolation). +6. **Verify.** Poll `127.0.0.1:3000` via ssh until healthy or timeout; on success also curl `https://` from the local machine (warn-only — TLS issuance or DNS may lag). Print logs and fail otherwise. +7. **Cleanup & report.** Prune `editable:*` images beyond the newest 3. Print the deployed tag, the site URL, and (first run) the local `.env` block for the data commands. + +## Repository changes + +1. `docker-compose.yml` — switch the service to `image: 'editable:${IMAGE_TAG:-local}'`; keep everything else (ports, env_file, bind mount) as is +2. `scripts/deploy_vps.sh` — the script per this spec (`set -euo pipefail`; remote steps as small quoted heredoc scripts, no unvalidated interpolation into remote shells) +3. `package.json` — `"deploy:vps": "./scripts/deploy_vps.sh"` for discoverability +4. `README.md` — rewrite Deploy to a VPS around the script; keep the manual compose flow as a short "doing it by hand" note +5. `.env.example` — update the deployment-block example values to the `editable-` container naming + +## Implementation steps + +1. Compose switch to `image:` + verify the local `docker compose up --build` path still works +2. Script skeleton: argument parsing, preflight, ssh helpers +3. Provision + configure phases against a throwaway droplet +4. Build/upload/activate/verify phases; end-to-end first deploy +5. Second-run path (update deploy), `--tag` rollback, image pruning +6. README rewrite + `package.json` script + +Each step is independently verifiable; 3–5 need a real amd64 VPS to test against. + +Status: implemented (compose switch, script, `deploy:vps` npm script, README, `.env.example`). Not yet tested against a real VPS — first-deploy, update-deploy, and `--tag` rollback runs on a throwaway droplet are the remaining verification. + +## Open questions + +- Should the script optionally harden the box (ufw allowing 22/80/443, unattended-upgrades)? Writebook's installer does some of this. Leaning yes for ufw, no for anything beyond — but deferred until after v1 works end to end. +- One site per server is the decided v1 scope. Multi-site later is additive, not a redesign — the collisions are: the fixed host port 3000, the hardcoded `container_name: editable`, the script owning all of `/etc/caddy/Caddyfile`, and the shared `editable:` image namespace. To keep that door open cheaply, v1 adopts two conventions now: name the container `editable-`, and write the Caddy config as `/etc/caddy/sites/.caddy` imported from the main Caddyfile. A future multi-site step then only needs a per-site `APP_PORT` and per-site image repos (`editable-:`). diff --git a/docker-compose.yml b/docker-compose.yml index a560afd7..72f47d39 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,12 +2,19 @@ # Nodion VPS, a home server. See README → Deploy to a VPS. Fly.io deployments # don't use this file (fly deploy builds the same Dockerfile directly). # +# Two ways to use it: +# - scripts/deploy_vps.sh uploads this file and starts a locally built image +# (IMAGE_TAG and CONTAINER_NAME come from the .deploy_env file it writes) +# - by hand on the server: `docker compose up -d --build` builds from source +# and runs under the default name `editable` +# # The app listens on 127.0.0.1:3000 — put a reverse proxy with TLS in front # (Caddy makes that two lines, see the README). services: editable: build: . - container_name: editable + image: 'editable:${IMAGE_TAG:-local}' + container_name: '${CONTAINER_NAME:-editable}' restart: unless-stopped ports: - '127.0.0.1:3000:3000' diff --git a/package.json b/package.json index 0ca48e2c..1eb0d85f 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "data:cloud-snapshots": "./scripts/data.sh cloud-snapshots", "data:verify": "./scripts/data.sh verify", "data:reset": "./scripts/data.sh reset", + "deploy:vps": "./scripts/deploy_vps.sh", "litestream:install": "./scripts/install-litestream.sh" }, "devDependencies": { diff --git a/scripts/deploy_vps.sh b/scripts/deploy_vps.sh new file mode 100755 index 00000000..9175e046 --- /dev/null +++ b/scripts/deploy_vps.sh @@ -0,0 +1,308 @@ +#!/usr/bin/env bash +# +# Deploy an Editable site to a VPS — see VPS_DEPLOY_SPEC.md for the design. +# +# One command takes a fresh Ubuntu server (amd64, ssh key access) to a running, +# TLS-terminated site; the same command run again ships a code update. The +# image is built locally and streamed over ssh — the server never needs git +# access, npm, or the memory to run a build. The only state that matters on +# the server is the bind-mounted data directory (/srv//data); the +# container is disposable and replaced on every deploy. +# +# Usage +# ./scripts/deploy_vps.sh [options] +# +# ./scripts/deploy_vps.sh root@203.0.113.10 my-site.example.com +# +# Options +# --tag deploy an already-uploaded image tag (rollback) instead of +# building — the last 3 tags are kept on the server +# --push-env re-sync ORIGIN and backup-bucket credentials into the +# server's .env (never done silently on normal deploys) +# --yes skip confirmation prompts (except the first-run password) +# +# Every run executes the same idempotent phases — preflight, provision, +# configure, build & upload, activate, verify, cleanup. The first run does +# everything; later runs fall through to the deploy phases in seconds. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR/.." + +die() { echo "Error: $*" >&2; exit 1; } +info() { echo "→ $*"; } +warn() { echo "Warning: $*" >&2; } + +confirm() { + [ "$YES" = true ] && return 0 + local reply + printf '%s [y/N] ' "$1" + read -r reply /dev/null || die "docker is not installed locally" +command -v git >/dev/null || die "git is not installed" +if [ -z "$TAG" ]; then + docker buildx version >/dev/null 2>&1 || die "docker buildx is not available" + git rev-parse --short HEAD >/dev/null 2>&1 || die "not inside a git checkout" +fi + +info "Checking ssh connectivity to $TARGET" +rssh true 2>/dev/null || die "cannot ssh into $TARGET (key-based access required)" + +REMOTE_ARCH="$(rssh uname -m)" +[ "$REMOTE_ARCH" = "x86_64" ] || + die "server is $REMOTE_ARCH — only amd64 servers are supported (the image pins the x86_64 Litestream build)" + +# DNS sanity check — warn only, propagation may still be in progress. +if echo "$HOST_ADDR" | grep -Eq '^[0-9.]+$' && command -v dig >/dev/null; then + RESOLVED="$(dig +short A "$DOMAIN" | sed -n '1p')" + if [ -z "$RESOLVED" ]; then + warn "$DOMAIN does not resolve yet — TLS will fail until the A record points at $HOST_ADDR" + elif [ "$RESOLVED" != "$HOST_ADDR" ]; then + warn "$DOMAIN resolves to $RESOLVED, not $HOST_ADDR — TLS will fail until DNS is fixed" + fi +fi + +FIRST_RUN=false +rssh "test -f $SRV/.env" 2>/dev/null || FIRST_RUN=true + +if [ "$FIRST_RUN" = true ]; then + confirm "Set up $TARGET as a new Editable server for https://$DOMAIN?" +fi + +# ---- phase 2: provision (remote, idempotent) --------------------------------- + +info "Provisioning server (no-op when already set up)" +{ + printf 'SITE=%q\nAPP_USER=%q\n' "$SITE" "$REMOTE_USER" + cat <<'REMOTE' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +command -v curl >/dev/null || { apt-get update -qq; apt-get install -y -qq curl ca-certificates; } +command -v docker >/dev/null || { echo "→ Installing Docker"; curl -fsSL https://get.docker.com | sh; } +command -v caddy >/dev/null || { + echo "→ Installing Caddy" + apt-get update -qq + apt-get install -y -qq debian-keyring debian-archive-keyring apt-transport-https gnupg + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | + gpg --dearmor --yes -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' \ + >/etc/apt/sources.list.d/caddy-stable.list + apt-get update -qq + apt-get install -y -qq caddy +} + +# 1 GB swap on small machines, so the running app never gets OOM-killed. +total_kb=$(awk '/MemTotal/ {print $2}' /proc/meminfo) +if [ "$total_kb" -lt 2000000 ] && [ "$(swapon --noheadings | wc -l)" -eq 0 ]; then + echo "→ Creating 1 GB swapfile (server has <2 GB RAM)" + fallocate -l 1G /swapfile + chmod 600 /swapfile + mkswap /swapfile >/dev/null + swapon /swapfile + grep -q '^/swapfile' /etc/fstab || echo '/swapfile none swap sw 0 0' >>/etc/fstab +fi + +mkdir -p "/srv/$SITE/data" +[ "$APP_USER" = "root" ] || chown "$APP_USER" "/srv/$SITE" +REMOTE +} | rssh "$SUDO bash -s" + +# ---- phase 3: configure (remote, idempotent) --------------------------------- + +info "Configuring Caddy for $DOMAIN" +{ + printf 'DOMAIN=%q\n' "$DOMAIN" + cat <<'REMOTE' +set -euo pipefail +mkdir -p /etc/caddy/sites +site_file="/etc/caddy/sites/$DOMAIN.caddy" +desired="$DOMAIN { + reverse_proxy 127.0.0.1:3000 +}" +changed="" +if [ ! -f "$site_file" ] || [ "$(cat "$site_file")" != "$desired" ]; then + printf '%s\n' "$desired" >"$site_file" + changed=1 +fi +grep -q 'import sites/\*\.caddy' /etc/caddy/Caddyfile || { + printf '\nimport sites/*.caddy\n' >>/etc/caddy/Caddyfile + changed=1 +} +[ -z "$changed" ] || systemctl reload caddy +REMOTE +} | rssh "$SUDO bash -s" + +# Read a value from the local .env (first uncommented occurrence). +local_env_val() { + [ -f .env ] || return 0 + sed -n -E "s/^$1=[\"']?([^\"']*)[\"']?[[:space:]]*$/\1/p" .env | sed -n '1p' +} + +BUCKET_KEYS="BUCKET_NAME AWS_ENDPOINT_URL_S3 AWS_REGION AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY" + +# Append KEY="value" for each bucket key with a local value to the file in $1. +append_bucket_creds() { + local key value + for key in $BUCKET_KEYS; do + value="$(local_env_val "$key")" + [ -n "$value" ] && printf '%s="%s"\n' "$key" "$value" >>"$1" + done + return 0 +} + +if [ "$FIRST_RUN" = true ]; then + info "Creating the server's .env" + GENERATED="$(openssl rand -base64 18 2>/dev/null || head -c 18 /dev/urandom | base64)" + printf 'Admin password for %s [enter for generated: %s]: ' "$DOMAIN" "$GENERATED" + read -r ADMIN_PW "$TMP/env" + append_bucket_creds "$TMP/env" + rssh "$SUDO tee $SRV/.env >/dev/null && $SUDO chmod 600 $SRV/.env" <"$TMP/env" +elif [ "$PUSH_ENV" = true ]; then + info "Re-syncing ORIGIN and bucket credentials into the server's .env" + rssh "$SUDO cat $SRV/.env" >"$TMP/env" + { + grep -E '^ADMIN_PASSWORD=' "$TMP/env" || die "server .env has no ADMIN_PASSWORD — refusing to rewrite it" + printf 'ORIGIN="https://%s"\n' "$DOMAIN" + } >"$TMP/env.new" + append_bucket_creds "$TMP/env.new" + echo "The server's .env will become (ADMIN_PASSWORD kept as is):" + sed -E 's/^(AWS_SECRET_ACCESS_KEY|ADMIN_PASSWORD)=.*/\1="…"/' "$TMP/env.new" | sed 's/^/ /' + confirm "Overwrite $SRV/.env with this?" + rssh "$SUDO tee $SRV/.env >/dev/null && $SUDO chmod 600 $SRV/.env" <"$TMP/env.new" +fi + +# ---- phase 4: build & upload ------------------------------------------------- + +if [ -z "$TAG" ]; then + TAG="$(git rev-parse --short HEAD)" + git diff-index --quiet HEAD -- 2>/dev/null || + warn "uncommitted changes — the image is built from the working tree but tagged $TAG" + info "Building editable:$TAG for linux/amd64" + docker buildx build --platform linux/amd64 -t "editable:$TAG" --load . + info "Streaming image to the server (this can take a few minutes)" + docker save "editable:$TAG" | gzip | rssh "gunzip | $SUDO docker load" +else + rssh "$SUDO docker image inspect editable:$TAG >/dev/null 2>&1" || + die "image editable:$TAG is not on the server (only built tags can be rolled back to)" + info "Deploying existing image editable:$TAG" +fi + +rssh "$SUDO tee $SRV/docker-compose.yml >/dev/null" /dev/null" +rssh "cd $SRV && $SUDO docker compose --env-file .env --env-file .deploy_env up -d --remove-orphans" + +# ---- phase 6: verify --------------------------------------------------------- + +info "Waiting for the app to respond" +if ! rssh 'for i in $(seq 30); do curl -fsS -o /dev/null http://127.0.0.1:3000 && exit 0; sleep 1; done; exit 1'; then + echo "--- container logs -----------------------------------------------------" >&2 + rssh "$SUDO docker logs --tail 50 $CONTAINER" >&2 || true + die "the app did not become healthy — the failing container is left running for inspection" +fi + +if ! curl -fsS -o /dev/null --max-time 15 "https://$DOMAIN" 2>/dev/null; then + warn "https://$DOMAIN is not reachable from here yet (DNS propagation or TLS issuance may still be in progress)" +fi + +# ---- phase 7: cleanup & report ----------------------------------------------- + +# Keep the newest 3 editable images for --tag rollbacks (rmi of an in-use +# image fails harmlessly, e.g. right after rolling back to an older tag). +rssh "$SUDO docker images editable --format '{{.Repository}}:{{.Tag}}' | tail -n +4 | xargs -r $SUDO docker rmi >/dev/null 2>&1 || true" + +echo +info "Deployed editable:$TAG to https://$DOMAIN" +if [ "$FIRST_RUN" = true ]; then + cat < Date: Mon, 20 Jul 2026 22:38:34 +0200 Subject: [PATCH 03/13] Harden deploy script --- scripts/deploy_vps.sh | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/deploy_vps.sh b/scripts/deploy_vps.sh index 9175e046..6744a356 100755 --- a/scripts/deploy_vps.sh +++ b/scripts/deploy_vps.sh @@ -102,6 +102,22 @@ SUDO="" SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new) rssh() { ssh "${SSH_OPTS[@]}" "$TARGET" "$@"; } +# Like rssh, but retries transient connection failures (ssh exit code 255) — +# fresh droplets drop occasional connections while they finish booting. Only +# for commands that read nothing from stdin (a retry would find it drained). +rssh_retry() { + local attempt rc + for attempt in 1 2 3 4 5; do + rc=0 + rssh "$@" /dev/null || die "cannot ssh into $TARGET (key-based access required)" +rssh_retry true 2>/dev/null || die "cannot ssh into $TARGET (key-based access required)" -REMOTE_ARCH="$(rssh uname -m)" +REMOTE_ARCH="$(rssh_retry uname -m)" [ "$REMOTE_ARCH" = "x86_64" ] || die "server is $REMOTE_ARCH — only amd64 servers are supported (the image pins the x86_64 Litestream build)" @@ -132,7 +148,7 @@ if echo "$HOST_ADDR" | grep -Eq '^[0-9.]+$' && command -v dig >/dev/null; then fi FIRST_RUN=false -rssh "test -f $SRV/.env" 2>/dev/null || FIRST_RUN=true +rssh_retry "test -f $SRV/.env" 2>/dev/null || FIRST_RUN=true if [ "$FIRST_RUN" = true ]; then confirm "Set up $TARGET as a new Editable server for https://$DOMAIN?" @@ -272,14 +288,14 @@ rssh "$SUDO tee $SRV/docker-compose.yml >/dev/null" /dev/null" -rssh "cd $SRV && $SUDO docker compose --env-file .env --env-file .deploy_env up -d --remove-orphans" +rssh_retry "cd $SRV && $SUDO docker compose --env-file .env --env-file .deploy_env up -d --remove-orphans" # ---- phase 6: verify --------------------------------------------------------- info "Waiting for the app to respond" -if ! rssh 'for i in $(seq 30); do curl -fsS -o /dev/null http://127.0.0.1:3000 && exit 0; sleep 1; done; exit 1'; then +if ! rssh_retry 'for i in $(seq 30); do curl -fsS -o /dev/null http://127.0.0.1:3000 && exit 0; sleep 1; done; exit 1'; then echo "--- container logs -----------------------------------------------------" >&2 - rssh "$SUDO docker logs --tail 50 $CONTAINER" >&2 || true + rssh_retry "$SUDO docker logs --tail 50 $CONTAINER" >&2 || true die "the app did not become healthy — the failing container is left running for inspection" fi @@ -291,7 +307,7 @@ fi # Keep the newest 3 editable images for --tag rollbacks (rmi of an in-use # image fails harmlessly, e.g. right after rolling back to an older tag). -rssh "$SUDO docker images editable --format '{{.Repository}}:{{.Tag}}' | tail -n +4 | xargs -r $SUDO docker rmi >/dev/null 2>&1 || true" +rssh_retry "$SUDO docker images editable --format '{{.Repository}}:{{.Tag}}' | tail -n +4 | xargs -r $SUDO docker rmi >/dev/null 2>&1 || true" echo info "Deployed editable:$TAG to https://$DOMAIN" From 0ea9a83f51eb69a3053a57ab4fbaad312ff41e75 Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Mon, 20 Jul 2026 22:49:11 +0200 Subject: [PATCH 04/13] Improve deploy script --- scripts/deploy_vps.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/deploy_vps.sh b/scripts/deploy_vps.sh index 6744a356..ed9385d4 100755 --- a/scripts/deploy_vps.sh +++ b/scripts/deploy_vps.sh @@ -97,9 +97,21 @@ HOST_ADDR="${TARGET#*@}" SUDO="" [ "$REMOTE_USER" = "root" ] || SUDO="sudo" +TMP="$(mktemp -d)" +# Close the multiplexing master connection, then remove the temp dir. +trap 'ssh -o ControlPath="$TMP/ssh" -O exit "$TARGET" 2>/dev/null || true; rm -rf "$TMP"' EXIT + # accept-new: trust a fresh server's host key on first contact (a brand-new # droplet is never in known_hosts), but still fail hard if a known key changes. -SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new) +# +# ControlMaster: multiplex every ssh call over one connection. A fresh VPS is +# hammered by ssh brute-force bots within minutes, and sshd (MaxStartups) +# randomly drops new incoming connections while bots fill the pending slots — +# with multiplexing only the first connection has to get through. +SSH_OPTS=( + -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new + -o ControlMaster=auto -o ControlPath="$TMP/ssh" -o ControlPersist=yes +) rssh() { ssh "${SSH_OPTS[@]}" "$TARGET" "$@"; } # Like rssh, but retries transient connection failures (ssh exit code 255) — @@ -118,9 +130,6 @@ rssh_retry() { return "$rc" } -TMP="$(mktemp -d)" -trap 'rm -rf "$TMP"' EXIT - # ---- phase 1: preflight ------------------------------------------------------ command -v docker >/dev/null || die "docker is not installed locally" @@ -293,7 +302,7 @@ rssh_retry "cd $SRV && $SUDO docker compose --env-file .env --env-file .deploy_e # ---- phase 6: verify --------------------------------------------------------- info "Waiting for the app to respond" -if ! rssh_retry 'for i in $(seq 30); do curl -fsS -o /dev/null http://127.0.0.1:3000 && exit 0; sleep 1; done; exit 1'; then +if ! rssh_retry 'for i in $(seq 30); do curl -fs -o /dev/null http://127.0.0.1:3000 && exit 0; sleep 1; done; exit 1'; then echo "--- container logs -----------------------------------------------------" >&2 rssh_retry "$SUDO docker logs --tail 50 $CONTAINER" >&2 || true die "the app did not become healthy — the failing container is left running for inspection" From 669b1b4a919f77c3e67e6cd94587a9004b8f836c Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Mon, 20 Jul 2026 23:00:01 +0200 Subject: [PATCH 05/13] Improve deploy script --- VPS_DEPLOY_SPEC.md | 2 ++ scripts/deploy_vps.sh | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/VPS_DEPLOY_SPEC.md b/VPS_DEPLOY_SPEC.md index f67600cf..6f0727e5 100644 --- a/VPS_DEPLOY_SPEC.md +++ b/VPS_DEPLOY_SPEC.md @@ -42,6 +42,8 @@ This file is the working spec for the feature. Once implemented and stable, fold **Idempotent provisioning, not a setup mode.** Every run executes the same phases; each phase checks before it acts (Docker installed? Caddy installed? Caddyfile current? `.env` present?). First run does everything; later runs fall through to the deploy phase in seconds. +**Dirty builds are labeled, not versioned.** A build from a working tree with uncommitted or untracked changes is tagged `-dirty`, so the image list and rollbacks can't mistake it for the committed state. Deploying dirty again reuses the tag — commits are the rollback anchors; dirty states are ephemeral by nature. + **Rollback = redeploy an old tag + existing data restore.** The script keeps the last 3 image tags on the server (older ones pruned after a successful deploy). `./scripts/deploy_vps.sh --tag ` starts that image instead of building. Content rollback is out of scope — that's `npm run data:restore`, which already works against the VPS via `DEPLOY_HOST`. **Health check gates success.** After `compose up`, the script polls `127.0.0.1:3000` over ssh (curl, ~30 s budget). On failure it prints the container logs and exits non-zero; it does not auto-rollback in v1. diff --git a/scripts/deploy_vps.sh b/scripts/deploy_vps.sh index ed9385d4..59a5752e 100755 --- a/scripts/deploy_vps.sh +++ b/scripts/deploy_vps.sh @@ -278,8 +278,13 @@ fi if [ -z "$TAG" ]; then TAG="$(git rev-parse --short HEAD)" - git diff-index --quiet HEAD -- 2>/dev/null || - warn "uncommitted changes — the image is built from the working tree but tagged $TAG" + # Uncommitted or untracked changes end up in the image — make the tag say + # so. Deploying dirty again reuses the tag (dirty states aren't versioned; + # commits are the rollback anchors). + if [ -n "$(git status --porcelain)" ]; then + TAG="$TAG-dirty" + warn "uncommitted changes — tagging the image $TAG" + fi info "Building editable:$TAG for linux/amd64" docker buildx build --platform linux/amd64 -t "editable:$TAG" --load . info "Streaming image to the server (this can take a few minutes)" From 939761c7f28dcca12d2c8c4d7cb3f141dbaeec83 Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Mon, 20 Jul 2026 23:30:00 +0200 Subject: [PATCH 06/13] Add commands for env var management --- README.md | 37 +++++--- VPS_DEPLOY_SPEC.md | 16 ++-- package.json | 3 +- scripts/deploy_vps.sh | 194 +++++++++++++++++++++++++++++++----------- 4 files changed, 183 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index ac6cee50..2027e489 100644 --- a/README.md +++ b/README.md @@ -697,18 +697,12 @@ Because each checkout manages exactly one app (see [Your site is your repo](#you Editable runs on any amd64 host with Docker — a DigitalOcean droplet, a Hetzner or Nodion VPS. One command takes a fresh Ubuntu server to a running site with TLS. Create the server with your ssh key installed, point your domain's A record at its IP, then: ```sh -npm run deploy:vps -- root@203.0.113.10 my-site.example.com +npm run vps:deploy -- root@203.0.113.10 my-site.example.com ``` -The first run provisions the server — Docker, [Caddy](https://caddyserver.com) as the TLS-terminating reverse proxy, swap on machines with less than 2 GB RAM — asks you for an admin password, builds the Docker image locally, streams it over ssh, and starts the site. The server never needs access to your git repository or the memory to run a build. Every later run of the same command ships an update: build, stream, replace the container, health-check. Your content lives in `/srv/my-site/data` on the server; deploys replace the container and never touch that folder. +The first run provisions the server — Docker, [Caddy](https://caddyserver.com) as the TLS-terminating reverse proxy, swap on machines with less than 2 GB RAM — asks you for an admin password, builds the Docker image locally, streams it over ssh, and starts the site. The server never needs access to your git repository or the memory to run a build. Your content lives in `/srv/my-site/data` on the server; deploys replace the container and never touch that folder. -Roll back a bad deploy by starting a previous image (the last three are kept on the server): - -```sh -npm run deploy:vps -- root@203.0.113.10 my-site.example.com --tag -``` - -After the first deploy the script prints the deployment block for your **local** `.env`, which points the data commands ([Backup, sync & recovery](#backup-sync--recovery)) at the server: +After the first deploy, the script prints the deployment block for your **local** `.env`. Add it — this is your checkout's deployment identity, playing the role `fly.toml` plays for Fly.io: ```sh DEPLOY_HOST="root@203.0.113.10" # who you ssh in as @@ -717,7 +711,30 @@ REMOTE_EXEC="docker exec editable-my-site" HOST_DATA_DIR="/srv/my-site/data" # the ./data bind mount, as an absolute path ``` -That's the entire switch: with `DEPLOY_HOST` set, every `npm run data:*` command targets the VPS (`fly.toml` is ignored); remove or comment it to target Fly.io again. Pull, push, backups, restores, point-in-time recovery — the whole toolbox behaves the same, including disaster recovery from the backup bucket on a fresh volume. For [automated backups](#automated-backups-optional), the `BUCKET_*` / `AWS_*` secrets belong in the server's `.env` — the script copies them from your local `.env` on the first deploy, and `--push-env` re-syncs them later. +With `DEPLOY_HOST` set, no command needs the server address anymore. Ship an update — build, stream, replace the container, health-check: + +```sh +npm run vps:deploy +``` + +Roll back a bad deploy by starting a previous image (the last three are kept on the server — `ssh` in and run `docker images editable` to list them; each tag is a git commit, so `git show -s ` tells you what's inside). A deploy from a working tree with uncommitted changes is tagged `-dirty` to keep experiments distinguishable from committed states: + +```sh +npm run vps:deploy -- --tag +``` + +And every `npm run data:*` command ([Backup, sync & recovery](#backup-sync--recovery)) targets the VPS too (`fly.toml` is ignored; remove or comment `DEPLOY_HOST` to target Fly.io again). Pull, push, backups, restores, point-in-time recovery — the whole toolbox behaves the same, including disaster recovery from the backup bucket on a fresh volume. + +The server keeps its own `.env` (the equivalent of `fly secrets`) — inspect and change it with the `env` command: + +```sh +npm run vps:env # show it (secrets masked) +npm run vps:env -- set BUCKET_NAME=my-backup AWS_REGION=auto +npm run vps:env -- set ADMIN_PASSWORD # no value = prompted, kept out of shell history +npm run vps:env -- unset BUCKET_NAME +``` + +`set` and `unset` restart the app so the change takes effect immediately. For [automated backups](#automated-backups-optional), the `BUCKET_*` / `AWS_*` secrets belong in the server's `.env` — the first deploy copies them from your local `.env` if present; after that, changes are explicit via `vps:env`. **Doing it by hand instead:** the script is optional — `docker-compose.yml` runs on any docker host. Clone your site on the server, `cp .env.example .env` and set `ADMIN_PASSWORD` and `ORIGIN="https://my-site.example.com"`, then `docker compose up -d --build`. The app listens on `127.0.0.1:3000`; put a reverse proxy with TLS in front (with Caddy that's the whole config: `my-site.example.com { reverse_proxy 127.0.0.1:3000 }`). Ship updates with `git pull && docker compose up -d --build`, and use `RESTART_CMD="docker restart editable"` / `REMOTE_EXEC="docker exec editable"` in the deployment block above (without the script, the container keeps the default name `editable`). diff --git a/VPS_DEPLOY_SPEC.md b/VPS_DEPLOY_SPEC.md index 6f0727e5..74b32f3e 100644 --- a/VPS_DEPLOY_SPEC.md +++ b/VPS_DEPLOY_SPEC.md @@ -38,7 +38,7 @@ This file is the working spec for the feature. Once implemented and stable, fold `/etc/caddy/Caddyfile` holds the reverse-proxy config. -**Secrets.** On first run the script prompts for `ADMIN_PASSWORD` (offering a generated one), sets `ORIGIN=https://`, and writes the server's `.env` over sftp. Backup-bucket credentials (`BUCKET_NAME`, `AWS_*`) are copied from the local `.env` if present. On later runs the server `.env` is left alone; a `--push-env` flag re-uploads the bucket/origin values (never silently — it prints what changes). Secrets are never passed as command-line arguments to remote shells. +**Secrets.** On first run the script prompts for `ADMIN_PASSWORD` (offering a generated one), sets `ORIGIN=https://`, and writes the server's `.env`. Backup-bucket credentials (`BUCKET_NAME`, `AWS_*`) are copied from the local `.env` if present. That first-run bootstrap is the only implicit sync: afterwards the server's `.env` is authoritative and changes only through the explicit `env` command (`env` show / `env set KEY=VALUE` / `env set KEY` with hidden prompt / `env unset KEY`), which rewrites the file and recreates the container so the change takes effect — fly-secrets style, nothing moves without being named. Secrets are never passed as command-line arguments to remote shells. **Idempotent provisioning, not a setup mode.** Every run executes the same phases; each phase checks before it acts (Docker installed? Caddy installed? Caddyfile current? `.env` present?). First run does everything; later runs fall through to the deploy phase in seconds. @@ -53,14 +53,20 @@ This file is the working spec for the feature. Once implemented and stable, fold ## Script interface ``` -./scripts/deploy_vps.sh [options] +./scripts/deploy_vps.sh first deploy, or explicit target (always works) +./scripts/deploy_vps.sh deploy to DEPLOY_HOST (npm run vps:deploy) +./scripts/deploy_vps.sh env show the server's env, masked (npm run vps:env) +./scripts/deploy_vps.sh env set KEY=VALUE … set env vars and restart the app +./scripts/deploy_vps.sh env set KEY prompt for the value (hidden input) +./scripts/deploy_vps.sh env unset KEY … remove env vars and restart the app Options: --tag deploy an already-uploaded image tag (rollback) instead of building - --push-env re-sync ORIGIN and bucket credentials into the server .env --yes skip confirmation prompts (except the first-run password prompt) ``` +**Addressing.** The short forms read `DEPLOY_HOST` from the local `.env` — the same key the data toolbox uses, playing the role `fly.toml` plays for Fly.io. The user adds it by hand from the block the first deploy prints (deliberately not auto-written, so it stays transparent where commands are targeted). The site and its domain are then discovered on the server (`/srv/*/.deploy_env`, exactly one per the one-site-per-server rule; the domain comes from `ORIGIN` in the server's `.env`). The explicit ` ` form is what works before any of that exists, and always overrides. + `` is typically `root@` on a fresh droplet; any sudo-capable user works. ssh key access is assumed (the script never handles passwords). ## Execution phases @@ -77,7 +83,7 @@ Options: 1. `docker-compose.yml` — switch the service to `image: 'editable:${IMAGE_TAG:-local}'`; keep everything else (ports, env_file, bind mount) as is 2. `scripts/deploy_vps.sh` — the script per this spec (`set -euo pipefail`; remote steps as small quoted heredoc scripts, no unvalidated interpolation into remote shells) -3. `package.json` — `"deploy:vps": "./scripts/deploy_vps.sh"` for discoverability +3. `package.json` — `"vps:deploy": "./scripts/deploy_vps.sh"` and `"vps:env": "./scripts/deploy_vps.sh env"` 4. `README.md` — rewrite Deploy to a VPS around the script; keep the manual compose flow as a short "doing it by hand" note 5. `.env.example` — update the deployment-block example values to the `editable-` container naming @@ -92,7 +98,7 @@ Options: Each step is independently verifiable; 3–5 need a real amd64 VPS to test against. -Status: implemented (compose switch, script, `deploy:vps` npm script, README, `.env.example`). Not yet tested against a real VPS — first-deploy, update-deploy, and `--tag` rollback runs on a throwaway droplet are the remaining verification. +Status: implemented (compose switch, script, `vps:deploy` / `vps:env` npm scripts, README, `.env.example`). Verified against a real DigitalOcean droplet: first deploy (provisioning, TLS via Caddy, password prompt), update deploy (cached build, container replacement, health check, report), and short-form `env` show with server-side site discovery. Real-world hardening that came out of that testing: ssh retries on transient connection drops (fresh droplets get hammered by brute-force bots, and sshd's MaxStartups randomly sheds new connections) and ssh connection multiplexing so each run plays that lottery only once. Still untested: `--tag` rollback, `env set`/`unset`, and the disaster-recovery path (fresh droplet restoring from a backup bucket). ## Open questions diff --git a/package.json b/package.json index 1eb0d85f..d37296d3 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,8 @@ "data:cloud-snapshots": "./scripts/data.sh cloud-snapshots", "data:verify": "./scripts/data.sh verify", "data:reset": "./scripts/data.sh reset", - "deploy:vps": "./scripts/deploy_vps.sh", + "vps:deploy": "./scripts/deploy_vps.sh", + "vps:env": "./scripts/deploy_vps.sh env", "litestream:install": "./scripts/install-litestream.sh" }, "devDependencies": { diff --git a/scripts/deploy_vps.sh b/scripts/deploy_vps.sh index 59a5752e..9570c4a6 100755 --- a/scripts/deploy_vps.sh +++ b/scripts/deploy_vps.sh @@ -10,18 +10,25 @@ # container is disposable and replaced on every deploy. # # Usage -# ./scripts/deploy_vps.sh [options] +# ./scripts/deploy_vps.sh first deploy (or explicit target) +# ./scripts/deploy_vps.sh deploy to DEPLOY_HOST +# ./scripts/deploy_vps.sh env show the server's env (secrets masked) +# ./scripts/deploy_vps.sh env set KEY=VALUE … set env vars and restart the app +# ./scripts/deploy_vps.sh env set KEY prompt for the value (hidden input) +# ./scripts/deploy_vps.sh env unset KEY … remove env vars and restart the app # -# ./scripts/deploy_vps.sh root@203.0.113.10 my-site.example.com +# The short forms read DEPLOY_HOST from your local .env — the deployment +# block printed after the first deploy, added by hand so it's transparent +# where the target comes from — and discover the site on the server. The +# explicit form works before that block exists and always overrides it: +# ./scripts/deploy_vps.sh root@203.0.113.10 my-site.example.com [env …] # # Options # --tag deploy an already-uploaded image tag (rollback) instead of # building — the last 3 tags are kept on the server -# --push-env re-sync ORIGIN and backup-bucket credentials into the -# server's .env (never done silently on normal deploys) # --yes skip confirmation prompts (except the first-run password) # -# Every run executes the same idempotent phases — preflight, provision, +# Every deploy runs the same idempotent phases — preflight, provision, # configure, build & upload, activate, verify, cleanup. The first run does # everything; later runs fall through to the deploy phases in seconds. set -euo pipefail @@ -43,13 +50,17 @@ confirm() { # ---- arguments --------------------------------------------------------------- -TARGET="" -DOMAIN="" TAG="" -PUSH_ENV=false YES=false +POSITIONAL=() -usage() { sed -n '3,26p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; } +usage() { awk 'NR < 3 { next } /^set -euo pipefail/ { exit } { sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}"; } + +# Read a value from the local .env (first uncommented occurrence). +local_env_val() { + [ -f .env ] || return 0 + sed -n -E "s/^$1=[\"']?([^\"']*)[\"']?[[:space:]]*$/\1/p" .env | sed -n '1p' +} while [ $# -gt 0 ]; do case "$1" in @@ -58,39 +69,59 @@ while [ $# -gt 0 ]; do TAG="$2" shift ;; - --push-env) PUSH_ENV=true ;; --yes) YES=true ;; -h | --help) usage; exit 0 ;; -*) die "Unknown option '$1' (see --help)" ;; - *) - if [ -z "$TARGET" ]; then TARGET="$1" - elif [ -z "$DOMAIN" ]; then DOMAIN="$1" - else die "Unexpected argument '$1' (see --help)" - fi - ;; + *) POSITIONAL+=("$1") ;; esac shift done -[ -n "$TARGET" ] && [ -n "$DOMAIN" ] || { usage; exit 2; } - -case "$TARGET" in - *@*) ;; - *) die "Target must be user@host, e.g. root@203.0.113.10" ;; +case "${POSITIONAL[0]:-}" in + *@*) + # Explicit form: [env …] + TARGET="${POSITIONAL[0]}" + DOMAIN="${POSITIONAL[1]:-}" + [ -n "$DOMAIN" ] || die "the explicit form needs a domain: deploy_vps.sh $TARGET " + ACTION="${POSITIONAL[2]:-deploy}" + ENV_ARGS=("${POSITIONAL[@]:3}") + case "$ACTION" in + deploy | env) ;; + *) die "Unknown command '$ACTION' (see --help)" ;; + esac + ;; + "" | env) + # Short form: the target comes from DEPLOY_HOST (environment wins over + # .env), the site and its domain are discovered on the server. + ACTION="${POSITIONAL[0]:-deploy}" + ENV_ARGS=("${POSITIONAL[@]:1}") + TARGET="${DEPLOY_HOST:-$(local_env_val DEPLOY_HOST)}" + [ -n "$TARGET" ] || die "DEPLOY_HOST is not set — add the deployment block to .env (printed after the first deploy), or pass an explicit target: deploy_vps.sh user@host domain" + case "$TARGET" in + *@*) ;; + *) die "DEPLOY_HOST must be user@host, e.g. root@203.0.113.10" ;; + esac + DOMAIN="" # discovered on the server + ;; + *) die "Unknown command '${POSITIONAL[0]}' (see --help)" ;; esac # Values interpolated into remote commands are validated to safe character # sets first — everything else reaches the remote side via stdin only. -DOMAIN="$(echo "$DOMAIN" | tr '[:upper:]' '[:lower:]')" -echo "$DOMAIN" | grep -Eq '^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$' || - die "'$DOMAIN' does not look like a domain name" +derive_site() { + DOMAIN="$(echo "$DOMAIN" | tr '[:upper:]' '[:lower:]')" + echo "$DOMAIN" | grep -Eq '^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$' || + die "'$DOMAIN' does not look like a domain name" + SITE="${DOMAIN%%.*}" + SRV="/srv/$SITE" + CONTAINER="editable-$SITE" +} +[ -z "$DOMAIN" ] || derive_site + if [ -n "$TAG" ]; then echo "$TAG" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9._-]*$' || die "'$TAG' is not a valid image tag" fi -SITE="${DOMAIN%%.*}" -SRV="/srv/$SITE" -CONTAINER="editable-$SITE" REMOTE_USER="${TARGET%%@*}" HOST_ADDR="${TARGET#*@}" @@ -132,16 +163,94 @@ rssh_retry() { # ---- phase 1: preflight ------------------------------------------------------ -command -v docker >/dev/null || die "docker is not installed locally" -command -v git >/dev/null || die "git is not installed" -if [ -z "$TAG" ]; then - docker buildx version >/dev/null 2>&1 || die "docker buildx is not available" - git rev-parse --short HEAD >/dev/null 2>&1 || die "not inside a git checkout" +if [ "$ACTION" = "deploy" ]; then + command -v docker >/dev/null || die "docker is not installed locally" + command -v git >/dev/null || die "git is not installed" + if [ -z "$TAG" ]; then + docker buildx version >/dev/null 2>&1 || die "docker buildx is not available" + git rev-parse --short HEAD >/dev/null 2>&1 || die "not inside a git checkout" + fi fi info "Checking ssh connectivity to $TARGET" rssh_retry true 2>/dev/null || die "cannot ssh into $TARGET (key-based access required)" +# Short form: discover the (single) site on the server and its domain. +if [ -z "$DOMAIN" ]; then + DEPLOYED="$(rssh_retry 'ls /srv/*/.deploy_env 2>/dev/null' || true)" + DEPLOYED_COUNT="$(printf '%s' "$DEPLOYED" | grep -c . || true)" + [ "$DEPLOYED_COUNT" -ge 1 ] || + die "no Editable deployment found on $TARGET — run the first deploy explicitly: deploy_vps.sh $TARGET " + [ "$DEPLOYED_COUNT" -eq 1 ] || + die "multiple sites found on $TARGET — address one explicitly: deploy_vps.sh $TARGET " + DISCOVERED_SITE="$(basename "$(dirname "$DEPLOYED")")" + echo "$DISCOVERED_SITE" | grep -Eq '^[a-z0-9]([a-z0-9-]*[a-z0-9])?$' || + die "unexpected site directory name '/srv/$DISCOVERED_SITE'" + DOMAIN="$(rssh_retry "$SUDO cat /srv/$DISCOVERED_SITE/.env" | sed -n 's|^ORIGIN="https://\([^"]*\)".*|\1|p' | sed -n '1p')" + [ -n "$DOMAIN" ] || die "could not read the site's domain (ORIGIN) from /srv/$DISCOVERED_SITE/.env" + derive_site + [ "$SITE" = "$DISCOVERED_SITE" ] || die "site directory /srv/$DISCOVERED_SITE does not match its ORIGIN domain $DOMAIN" + info "Site: $DOMAIN ($TARGET)" +fi + +# ---- env command ------------------------------------------------------------- +# Explicit env var management, fly-secrets style: show / set / unset. Changes +# take effect by recreating the container (env_file is baked in at creation). + +restart_with_env() { + rssh "$SUDO tee $SRV/.env >/dev/null && $SUDO chmod 600 $SRV/.env" <"$TMP/env" + info "Restarting the app with the new environment" + rssh_retry "cd $SRV && $SUDO docker compose --env-file .env --env-file .deploy_env up -d" + rssh_retry 'for i in $(seq 30); do curl -fs -o /dev/null http://127.0.0.1:3000 && exit 0; sleep 1; done; exit 1' || + die "the app did not come back healthy — inspect with: ssh $TARGET '$SUDO docker logs $CONTAINER'" +} + +if [ "$ACTION" = "env" ]; then + rssh "test -f $SRV/.env && test -f $SRV/.deploy_env" 2>/dev/null || + die "no deployment at $SRV yet — deploy first" + rssh "$SUDO cat $SRV/.env" >"$TMP/env" + SUB="${ENV_ARGS[0]:-show}" + case "$SUB" in + show) + sed -E 's/^(ADMIN_PASSWORD|AWS_SECRET_ACCESS_KEY)=.*/\1="…"/' "$TMP/env" + ;; + set) + [ ${#ENV_ARGS[@]} -ge 2 ] || die "env set needs at least one KEY=VALUE (or KEY to be prompted)" + for pair in "${ENV_ARGS[@]:1}"; do + key="${pair%%=*}" + echo "$key" | grep -Eq '^[A-Z][A-Z0-9_]*$' || die "'$key' is not a valid env var name" + if [ "$pair" = "$key" ]; then + printf 'Value for %s (input hidden): ' "$key" + read -rs value "$TMP/env.new" || true + printf '%s="%s"\n' "$key" "$value" >>"$TMP/env.new" + mv "$TMP/env.new" "$TMP/env" + info "Set $key" + done + restart_with_env + ;; + unset) + [ ${#ENV_ARGS[@]} -ge 2 ] || die "env unset needs at least one KEY" + for key in "${ENV_ARGS[@]:1}"; do + grep -q "^$key=" "$TMP/env" || die "$key is not set on the server" + grep -v "^$key=" "$TMP/env" >"$TMP/env.new" || true + mv "$TMP/env.new" "$TMP/env" + info "Unset $key" + done + restart_with_env + ;; + *) die "Unknown env command '$SUB' (expected: show, set, unset)" ;; + esac + exit 0 +fi + REMOTE_ARCH="$(rssh_retry uname -m)" [ "$REMOTE_ARCH" = "x86_64" ] || die "server is $REMOTE_ARCH — only amd64 servers are supported (the image pins the x86_64 Litestream build)" @@ -227,12 +336,6 @@ grep -q 'import sites/\*\.caddy' /etc/caddy/Caddyfile || { REMOTE } | rssh "$SUDO bash -s" -# Read a value from the local .env (first uncommented occurrence). -local_env_val() { - [ -f .env ] || return 0 - sed -n -E "s/^$1=[\"']?([^\"']*)[\"']?[[:space:]]*$/\1/p" .env | sed -n '1p' -} - BUCKET_KEYS="BUCKET_NAME AWS_ENDPOINT_URL_S3 AWS_REGION AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY" # Append KEY="value" for each bucket key with a local value to the file in $1. @@ -260,18 +363,6 @@ if [ "$FIRST_RUN" = true ]; then } >"$TMP/env" append_bucket_creds "$TMP/env" rssh "$SUDO tee $SRV/.env >/dev/null && $SUDO chmod 600 $SRV/.env" <"$TMP/env" -elif [ "$PUSH_ENV" = true ]; then - info "Re-syncing ORIGIN and bucket credentials into the server's .env" - rssh "$SUDO cat $SRV/.env" >"$TMP/env" - { - grep -E '^ADMIN_PASSWORD=' "$TMP/env" || die "server .env has no ADMIN_PASSWORD — refusing to rewrite it" - printf 'ORIGIN="https://%s"\n' "$DOMAIN" - } >"$TMP/env.new" - append_bucket_creds "$TMP/env.new" - echo "The server's .env will become (ADMIN_PASSWORD kept as is):" - sed -E 's/^(AWS_SECRET_ACCESS_KEY|ADMIN_PASSWORD)=.*/\1="…"/' "$TMP/env.new" | sed 's/^/ /' - confirm "Overwrite $SRV/.env with this?" - rssh "$SUDO tee $SRV/.env >/dev/null && $SUDO chmod 600 $SRV/.env" <"$TMP/env.new" fi # ---- phase 4: build & upload ------------------------------------------------- @@ -328,7 +419,8 @@ info "Deployed editable:$TAG to https://$DOMAIN" if [ "$FIRST_RUN" = true ]; then cat < Date: Mon, 20 Jul 2026 23:43:51 +0200 Subject: [PATCH 07/13] Add vps:status command --- README.md | 14 +++++++++++++- VPS_DEPLOY_SPEC.md | 3 ++- package.json | 1 + scripts/deploy_vps.sh | 43 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2027e489..674ab7da 100644 --- a/README.md +++ b/README.md @@ -717,7 +717,19 @@ With `DEPLOY_HOST` set, no command needs the server address anymore. Ship an upd npm run vps:deploy ``` -Roll back a bad deploy by starting a previous image (the last three are kept on the server — `ssh` in and run `docker images editable` to list them; each tag is a git commit, so `git show -s ` tells you what's inside). A deploy from a working tree with uncommitted changes is tagged `-dirty` to keep experiments distinguishable from committed states: +See what's running and what you can roll back to — each image tag is a git commit, and the last three are kept on the server (a deploy from a working tree with uncommitted changes is tagged `-dirty` to keep experiments distinguishable from committed states): + +```sh +npm run vps:status + +# → Running: editable:939761c (Up 5 minutes) — https://my-site.example.com +# +# Images on the server (newest first): +# 939761c 2026-07-20 21:35 Add env var management ← running +# 502dcdf 2026-07-20 20:43 Harden deploy script +``` + +Roll back a bad deploy by starting a previous image: ```sh npm run vps:deploy -- --tag diff --git a/VPS_DEPLOY_SPEC.md b/VPS_DEPLOY_SPEC.md index 74b32f3e..6c5c7eba 100644 --- a/VPS_DEPLOY_SPEC.md +++ b/VPS_DEPLOY_SPEC.md @@ -55,6 +55,7 @@ This file is the working spec for the feature. Once implemented and stable, fold ``` ./scripts/deploy_vps.sh first deploy, or explicit target (always works) ./scripts/deploy_vps.sh deploy to DEPLOY_HOST (npm run vps:deploy) +./scripts/deploy_vps.sh status running tag + rollback tags (npm run vps:status) ./scripts/deploy_vps.sh env show the server's env, masked (npm run vps:env) ./scripts/deploy_vps.sh env set KEY=VALUE … set env vars and restart the app ./scripts/deploy_vps.sh env set KEY prompt for the value (hidden input) @@ -98,7 +99,7 @@ Options: Each step is independently verifiable; 3–5 need a real amd64 VPS to test against. -Status: implemented (compose switch, script, `vps:deploy` / `vps:env` npm scripts, README, `.env.example`). Verified against a real DigitalOcean droplet: first deploy (provisioning, TLS via Caddy, password prompt), update deploy (cached build, container replacement, health check, report), and short-form `env` show with server-side site discovery. Real-world hardening that came out of that testing: ssh retries on transient connection drops (fresh droplets get hammered by brute-force bots, and sshd's MaxStartups randomly sheds new connections) and ssh connection multiplexing so each run plays that lottery only once. Still untested: `--tag` rollback, `env set`/`unset`, and the disaster-recovery path (fresh droplet restoring from a backup bucket). +Status: implemented (compose switch, script, `vps:deploy` / `vps:env` npm scripts, README, `.env.example`). Verified against a real DigitalOcean droplet: first deploy (provisioning, TLS via Caddy, password prompt), update deploy (cached build, container replacement, health check, report), and the short-form `env` show and `status` commands with server-side site discovery. Real-world hardening that came out of that testing: ssh retries on transient connection drops (fresh droplets get hammered by brute-force bots, and sshd's MaxStartups randomly sheds new connections) and ssh connection multiplexing so each run plays that lottery only once. Still untested: `--tag` rollback, `env set`/`unset`, and the disaster-recovery path (fresh droplet restoring from a backup bucket). ## Open questions diff --git a/package.json b/package.json index d37296d3..985bc990 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "data:reset": "./scripts/data.sh reset", "vps:deploy": "./scripts/deploy_vps.sh", "vps:env": "./scripts/deploy_vps.sh env", + "vps:status": "./scripts/deploy_vps.sh status", "litestream:install": "./scripts/install-litestream.sh" }, "devDependencies": { diff --git a/scripts/deploy_vps.sh b/scripts/deploy_vps.sh index 9570c4a6..7accfccf 100755 --- a/scripts/deploy_vps.sh +++ b/scripts/deploy_vps.sh @@ -12,6 +12,7 @@ # Usage # ./scripts/deploy_vps.sh first deploy (or explicit target) # ./scripts/deploy_vps.sh deploy to DEPLOY_HOST +# ./scripts/deploy_vps.sh status show the running tag and rollback candidates # ./scripts/deploy_vps.sh env show the server's env (secrets masked) # ./scripts/deploy_vps.sh env set KEY=VALUE … set env vars and restart the app # ./scripts/deploy_vps.sh env set KEY prompt for the value (hidden input) @@ -86,11 +87,11 @@ case "${POSITIONAL[0]:-}" in ACTION="${POSITIONAL[2]:-deploy}" ENV_ARGS=("${POSITIONAL[@]:3}") case "$ACTION" in - deploy | env) ;; + deploy | env | status) ;; *) die "Unknown command '$ACTION' (see --help)" ;; esac ;; - "" | env) + "" | env | status) # Short form: the target comes from DEPLOY_HOST (environment wins over # .env), the site and its domain are discovered on the server. ACTION="${POSITIONAL[0]:-deploy}" @@ -193,6 +194,44 @@ if [ -z "$DOMAIN" ]; then info "Site: $DOMAIN ($TARGET)" fi +# ---- status command ---------------------------------------------------------- +# What's running and what can be rolled back to — the tags live on the server, +# the commit messages behind them live in this checkout's git history. + +if [ "$ACTION" = "status" ]; then + { + printf 'CONTAINER=%q\n' "$CONTAINER" + cat <<'REMOTE' +docker ps --filter "name=^$CONTAINER$" --format '{{.Image}}|{{.Status}}' +echo --- +docker images editable --format '{{.Tag}}|{{.CreatedAt}}' +REMOTE + } | rssh "$SUDO bash -s" >"$TMP/status" + + RUNNING_IMG="$(awk -F'|' '/^---$/ { exit } { print $1; exit }' "$TMP/status")" + RUNNING_STATE="$(awk -F'|' '/^---$/ { exit } { print $2; exit }' "$TMP/status")" + if [ -n "$RUNNING_IMG" ]; then + info "Running: $RUNNING_IMG ($RUNNING_STATE) — https://$DOMAIN" + else + warn "no running container named $CONTAINER" + fi + + echo + echo "Images on the server (newest first):" + awk 'found { print } /^---$/ { found = 1 }' "$TMP/status" | + while IFS='|' read -r tag created; do + base_tag="${tag%-dirty}" + desc="$(git show -s --format=%s "$base_tag" 2>/dev/null || echo '(not a commit in this checkout)')" + [ "$tag" = "$base_tag" ] || desc="$desc + uncommitted changes" + marker="" + [ "editable:$tag" = "$RUNNING_IMG" ] && marker=" ← running" + printf ' %-16s %.16s %s%s\n' "$tag" "$created" "$desc" "$marker" + done + echo + echo "Roll back with: npm run vps:deploy -- --tag " + exit 0 +fi + # ---- env command ------------------------------------------------------------- # Explicit env var management, fly-secrets style: show / set / unset. Changes # take effect by recreating the container (env_file is baked in at creation). From b8d67e37116bfc61876765f79110488e00d94db9 Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Mon, 20 Jul 2026 23:55:26 +0200 Subject: [PATCH 08/13] Simplify --- .env.example | 15 +++++++++------ README.md | 7 ++----- VPS_DEPLOY_SPEC.md | 4 ++-- docker-compose.yml | 4 ++++ scripts/data.sh | 25 ++++++++++++++++++++++++- scripts/deploy_vps.sh | 8 +++----- 6 files changed, 44 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index 925f120e..2639efda 100644 --- a/.env.example +++ b/.env.example @@ -7,11 +7,14 @@ ADMIN_PASSWORD="your-secure-password-here" # AWS_ACCESS_KEY_ID="" # AWS_SECRET_ACCESS_KEY="" -# Generic VPS deployment (optional) — targets the npm run data:* commands at -# a host reachable over plain ssh instead of Fly.io. scripts/deploy_vps.sh -# prints this block with the right values after the first deploy; for a -# hand-managed compose setup the container is named plain "editable". +# Generic VPS deployment (optional) — targets the npm run vps:* and +# npm run data:* commands at a host reachable over plain ssh instead of +# Fly.io. For a server managed by scripts/deploy_vps.sh this one line is all +# that's needed (the rest is discovered from the server): # DEPLOY_HOST="root@my-site.example.com" -# RESTART_CMD="docker restart editable-my-site" -# REMOTE_EXEC="docker exec editable-my-site" +# +# Only for hand-managed setups without that script (bare node, own compose — +# nothing to discover), or as overrides: +# RESTART_CMD="docker restart editable" +# REMOTE_EXEC="docker exec editable" # HOST_DATA_DIR="/srv/my-site/data" diff --git a/README.md b/README.md index 674ab7da..d9a39c3e 100644 --- a/README.md +++ b/README.md @@ -702,13 +702,10 @@ npm run vps:deploy -- root@203.0.113.10 my-site.example.com The first run provisions the server — Docker, [Caddy](https://caddyserver.com) as the TLS-terminating reverse proxy, swap on machines with less than 2 GB RAM — asks you for an admin password, builds the Docker image locally, streams it over ssh, and starts the site. The server never needs access to your git repository or the memory to run a build. Your content lives in `/srv/my-site/data` on the server; deploys replace the container and never touch that folder. -After the first deploy, the script prints the deployment block for your **local** `.env`. Add it — this is your checkout's deployment identity, playing the role `fly.toml` plays for Fly.io: +After the first deploy, add one line to your **local** `.env` — this is your checkout's deployment identity, playing the role `fly.toml` plays for Fly.io. Everything else (container name, data path) is read from the server: ```sh DEPLOY_HOST="root@203.0.113.10" # who you ssh in as -RESTART_CMD="docker restart editable-my-site" -REMOTE_EXEC="docker exec editable-my-site" -HOST_DATA_DIR="/srv/my-site/data" # the ./data bind mount, as an absolute path ``` With `DEPLOY_HOST` set, no command needs the server address anymore. Ship an update — build, stream, replace the container, health-check: @@ -748,7 +745,7 @@ npm run vps:env -- unset BUCKET_NAME `set` and `unset` restart the app so the change takes effect immediately. For [automated backups](#automated-backups-optional), the `BUCKET_*` / `AWS_*` secrets belong in the server's `.env` — the first deploy copies them from your local `.env` if present; after that, changes are explicit via `vps:env`. -**Doing it by hand instead:** the script is optional — `docker-compose.yml` runs on any docker host. Clone your site on the server, `cp .env.example .env` and set `ADMIN_PASSWORD` and `ORIGIN="https://my-site.example.com"`, then `docker compose up -d --build`. The app listens on `127.0.0.1:3000`; put a reverse proxy with TLS in front (with Caddy that's the whole config: `my-site.example.com { reverse_proxy 127.0.0.1:3000 }`). Ship updates with `git pull && docker compose up -d --build`, and use `RESTART_CMD="docker restart editable"` / `REMOTE_EXEC="docker exec editable"` in the deployment block above (without the script, the container keeps the default name `editable`). +**Doing it by hand instead:** the script is optional — `docker-compose.yml` runs on any docker host. Clone your site on the server, `cp .env.example .env` and set `ADMIN_PASSWORD` and `ORIGIN="https://my-site.example.com"`, then `docker compose up -d --build`. The app listens on `127.0.0.1:3000`; put a reverse proxy with TLS in front (with Caddy that's the whole config: `my-site.example.com { reverse_proxy 127.0.0.1:3000 }`). Ship updates with `git pull && docker compose up -d --build`. For the data commands, a hand-managed setup has nothing to auto-discover, so set the explicit keys alongside `DEPLOY_HOST` in your local `.env`: `RESTART_CMD="docker restart editable"`, `REMOTE_EXEC="docker exec editable"`, and `HOST_DATA_DIR` pointing at the `./data` bind mount as an absolute path (without the script, the container keeps the default name `editable`). What Fly.io still does for you that a VPS doesn't: scale-to-zero with sub-second wake-ups (a VPS runs — and bills — around the clock), TLS and anycast routing without a reverse proxy, and volume snapshots. The VPS path trades that for a fixed monthly price and no platform dependency. diff --git a/VPS_DEPLOY_SPEC.md b/VPS_DEPLOY_SPEC.md index 6c5c7eba..5a9003ce 100644 --- a/VPS_DEPLOY_SPEC.md +++ b/VPS_DEPLOY_SPEC.md @@ -66,7 +66,7 @@ Options: --yes skip confirmation prompts (except the first-run password prompt) ``` -**Addressing.** The short forms read `DEPLOY_HOST` from the local `.env` — the same key the data toolbox uses, playing the role `fly.toml` plays for Fly.io. The user adds it by hand from the block the first deploy prints (deliberately not auto-written, so it stays transparent where commands are targeted). The site and its domain are then discovered on the server (`/srv/*/.deploy_env`, exactly one per the one-site-per-server rule; the domain comes from `ORIGIN` in the server's `.env`). The explicit ` ` form is what works before any of that exists, and always overrides. +**Addressing.** `DEPLOY_HOST` in the local `.env` is the checkout's entire deployment identity — the same key the data toolbox uses, playing the role `fly.toml` plays for Fly.io. The user adds it by hand from the line the first deploy prints (deliberately not auto-written, so it stays transparent where commands are targeted). Everything else is discovered on the server via the `/srv//.deploy_env` marker (exactly one per the one-site-per-server rule): the deploy script reads the site and its domain (from `ORIGIN` in the server's `.env`), and `data.sh` derives `REMOTE_EXEC`, `RESTART_CMD`, and `HOST_DATA_DIR` from the marker's `CONTAINER_NAME` and path. Explicit values always override, and setups without the marker (bare node, hand-managed compose) keep configuring the data toolbox explicitly. The explicit ` ` form is what works before any of this exists. `` is typically `root@` on a fresh droplet; any sudo-capable user works. ssh key access is assumed (the script never handles passwords). @@ -99,7 +99,7 @@ Options: Each step is independently verifiable; 3–5 need a real amd64 VPS to test against. -Status: implemented (compose switch, script, `vps:deploy` / `vps:env` npm scripts, README, `.env.example`). Verified against a real DigitalOcean droplet: first deploy (provisioning, TLS via Caddy, password prompt), update deploy (cached build, container replacement, health check, report), and the short-form `env` show and `status` commands with server-side site discovery. Real-world hardening that came out of that testing: ssh retries on transient connection drops (fresh droplets get hammered by brute-force bots, and sshd's MaxStartups randomly sheds new connections) and ssh connection multiplexing so each run plays that lottery only once. Still untested: `--tag` rollback, `env set`/`unset`, and the disaster-recovery path (fresh droplet restoring from a backup bucket). +Status: implemented (compose switch, script, `vps:deploy` / `vps:env` / `vps:status` npm scripts, `data.sh` auto-discovery, README, `.env.example`). Verified against a real DigitalOcean droplet: first deploy (provisioning, TLS via Caddy, password prompt), update deploy (cached build, container replacement, health check, report), and the short-form `env` show and `status` commands with server-side site discovery. Real-world hardening that came out of that testing: ssh retries on transient connection drops (fresh droplets get hammered by brute-force bots, and sshd's MaxStartups randomly sheds new connections) and ssh connection multiplexing so each run plays that lottery only once. Also verified: `--tag` deploys (used to ship a compose-only fix without rebuilding) and `data.sh` against a script-managed server with only `DEPLOY_HOST` set. That testing surfaced and fixed a pre-existing bug: `docker-compose.yml` never set `DATA_DIR=/data` (fly.toml does for Fly), so compose-run containers wrote content to the ephemeral `/app/data` and every redeploy silently wiped it — now fixed in the compose file. Still untested: `env set`/`unset` and the disaster-recovery path (fresh droplet restoring from a backup bucket). ## Open questions diff --git a/docker-compose.yml b/docker-compose.yml index 72f47d39..b1510ee8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,10 @@ services: - '127.0.0.1:3000:3000' env_file: .env environment: + # Point the app at the bind mount — without this it defaults to ./data + # inside the container and content silently dies with each deploy + # (fly.toml sets the same for Fly deployments). + DATA_DIR: /data BODY_SIZE_LIMIT: '${BODY_SIZE_LIMIT:-30000000}' volumes: - ./data:/data diff --git a/scripts/data.sh b/scripts/data.sh index ac80528d..a38aaa9f 100755 --- a/scripts/data.sh +++ b/scripts/data.sh @@ -36,7 +36,10 @@ # Fly driver: the target app is read from fly.toml (app = '...'), same as the # fly CLI. Override with -a or the FLY_APP environment variable (-a wins). # -# Ssh driver: configured via environment variables or .env (environment wins): +# Ssh driver: configured via environment variables or .env (environment wins). +# For a server managed by scripts/deploy_vps.sh, DEPLOY_HOST is the only key +# needed — the rest is discovered from the server. The explicit keys are for +# other setups (bare node, hand-managed compose) and always override: # DEPLOY_HOST user@host to ssh into (setting this selects the driver) # RESTART_CMD how to restart the app, e.g. 'docker restart editable' # REMOTE_EXEC command prefix to enter the app context, e.g. @@ -106,6 +109,26 @@ fi DEPLOY_HOST="${DEPLOY_HOST:-}" REMOTE_APP_DIR="${REMOTE_APP_DIR:-/app}" REMOTE_DATA="${REMOTE_DATA_DIR:-/data}" + +# A deploy_vps.sh-managed server needs only DEPLOY_HOST — container name and +# data path are discovered from its /srv//.deploy_env marker, the way +# fly.toml resolves the rest for the fly driver. Explicit values always win, +# and without exactly one marker (bare node, hand-managed compose, multiple +# sites) nothing changes and the explicit keys below stay required. +if [ "$DRIVER" = "ssh" ] && [ -n "$DEPLOY_HOST" ] && + [ -z "${REMOTE_EXEC:-}" ] && [ -z "${RESTART_CMD:-}" ] && [ -z "${HOST_DATA_DIR:-}" ]; then + FOUND="$(ssh "$DEPLOY_HOST" 'for f in /srv/*/.deploy_env; do [ -f "$f" ] && printf "%s %s\n" "$f" "$(sed -n s/^CONTAINER_NAME=//p "$f")"; done' 2>/dev/null || true)" + if [ "$(printf '%s' "$FOUND" | grep -c .)" -eq 1 ]; then + MARKER="${FOUND%% *}" + CONTAINER="${FOUND#* }" + if printf '%s' "$CONTAINER" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9._-]*$'; then + REMOTE_EXEC="docker exec $CONTAINER" + RESTART_CMD="docker restart $CONTAINER" + HOST_DATA_DIR="$(dirname "$MARKER")/data" + fi + fi +fi + HOST_DATA_DIR="${HOST_DATA_DIR:-$REMOTE_DATA}" REMOTE_EXEC="${REMOTE_EXEC:-}" diff --git a/scripts/deploy_vps.sh b/scripts/deploy_vps.sh index 7accfccf..03d5b4a8 100755 --- a/scripts/deploy_vps.sh +++ b/scripts/deploy_vps.sh @@ -458,12 +458,10 @@ info "Deployed editable:$TAG to https://$DOMAIN" if [ "$FIRST_RUN" = true ]; then cat < Date: Tue, 21 Jul 2026 00:10:21 +0200 Subject: [PATCH 09/13] Rename to vps-deploy.sh --- .env.example | 2 +- VPS_DEPLOY_SPEC.md | 24 ++++++++++++------------ docker-compose.yml | 2 +- package.json | 6 +++--- scripts/data.sh | 4 ++-- scripts/{deploy_vps.sh => vps-deploy.sh} | 24 ++++++++++++------------ 6 files changed, 31 insertions(+), 31 deletions(-) rename scripts/{deploy_vps.sh => vps-deploy.sh} (95%) diff --git a/.env.example b/.env.example index 2639efda..85e61284 100644 --- a/.env.example +++ b/.env.example @@ -9,7 +9,7 @@ ADMIN_PASSWORD="your-secure-password-here" # Generic VPS deployment (optional) — targets the npm run vps:* and # npm run data:* commands at a host reachable over plain ssh instead of -# Fly.io. For a server managed by scripts/deploy_vps.sh this one line is all +# Fly.io. For a server managed by scripts/vps-deploy.sh this one line is all # that's needed (the rest is discovered from the server): # DEPLOY_HOST="root@my-site.example.com" # diff --git a/VPS_DEPLOY_SPEC.md b/VPS_DEPLOY_SPEC.md index 5a9003ce..814d9be7 100644 --- a/VPS_DEPLOY_SPEC.md +++ b/VPS_DEPLOY_SPEC.md @@ -1,12 +1,12 @@ # VPS deploy script — specification -Spec for `scripts/deploy_vps.sh`: a single local script that takes a fresh Ubuntu VPS (e.g. a DigitalOcean droplet) from zero to a running, TLS-terminated Editable site, and afterwards ships code updates to the same box. Modeled on the Writebook/ONCE installer experience, without Kamal or any registry. +Spec for `scripts/vps-deploy.sh`: a single local script that takes a fresh Ubuntu VPS (e.g. a DigitalOcean droplet) from zero to a running, TLS-terminated Editable site, and afterwards ships code updates to the same box. Modeled on the Writebook/ONCE installer experience, without Kamal or any registry. This file is the working spec for the feature. Once implemented and stable, fold the design decisions into `ARCHITECTURE.md` and the user-facing instructions into README → Deploy to a VPS. ## Goals -- One command against a fresh VPS sets up everything: `./scripts/deploy_vps.sh root@203.0.113.10 my-site.example.com` +- One command against a fresh VPS sets up everything: `./scripts/vps-deploy.sh root@203.0.113.10 my-site.example.com` - The same command run again ships a code update (the script detects what's needed; no separate setup/deploy modes for the user to learn) - The image is built locally and streamed over ssh — the server never needs git access, npm, or the memory to run a Vite build - The only state on the server that matters is the bind-mounted data directory; the container is disposable and replaced on every deploy @@ -44,7 +44,7 @@ This file is the working spec for the feature. Once implemented and stable, fold **Dirty builds are labeled, not versioned.** A build from a working tree with uncommitted or untracked changes is tagged `-dirty`, so the image list and rollbacks can't mistake it for the committed state. Deploying dirty again reuses the tag — commits are the rollback anchors; dirty states are ephemeral by nature. -**Rollback = redeploy an old tag + existing data restore.** The script keeps the last 3 image tags on the server (older ones pruned after a successful deploy). `./scripts/deploy_vps.sh --tag ` starts that image instead of building. Content rollback is out of scope — that's `npm run data:restore`, which already works against the VPS via `DEPLOY_HOST`. +**Rollback = redeploy an old tag + existing data restore.** The script keeps the last 3 image tags on the server (older ones pruned after a successful deploy). `./scripts/vps-deploy.sh --tag ` starts that image instead of building. Content rollback is out of scope — that's `npm run data:restore`, which already works against the VPS via `DEPLOY_HOST`. **Health check gates success.** After `compose up`, the script polls `127.0.0.1:3000` over ssh (curl, ~30 s budget). On failure it prints the container logs and exits non-zero; it does not auto-rollback in v1. @@ -53,13 +53,13 @@ This file is the working spec for the feature. Once implemented and stable, fold ## Script interface ``` -./scripts/deploy_vps.sh first deploy, or explicit target (always works) -./scripts/deploy_vps.sh deploy to DEPLOY_HOST (npm run vps:deploy) -./scripts/deploy_vps.sh status running tag + rollback tags (npm run vps:status) -./scripts/deploy_vps.sh env show the server's env, masked (npm run vps:env) -./scripts/deploy_vps.sh env set KEY=VALUE … set env vars and restart the app -./scripts/deploy_vps.sh env set KEY prompt for the value (hidden input) -./scripts/deploy_vps.sh env unset KEY … remove env vars and restart the app +./scripts/vps-deploy.sh first deploy, or explicit target (always works) +./scripts/vps-deploy.sh deploy to DEPLOY_HOST (npm run vps:deploy) +./scripts/vps-deploy.sh status running tag + rollback tags (npm run vps:status) +./scripts/vps-deploy.sh env show the server's env, masked (npm run vps:env) +./scripts/vps-deploy.sh env set KEY=VALUE … set env vars and restart the app +./scripts/vps-deploy.sh env set KEY prompt for the value (hidden input) +./scripts/vps-deploy.sh env unset KEY … remove env vars and restart the app Options: --tag deploy an already-uploaded image tag (rollback) instead of building @@ -83,8 +83,8 @@ Options: ## Repository changes 1. `docker-compose.yml` — switch the service to `image: 'editable:${IMAGE_TAG:-local}'`; keep everything else (ports, env_file, bind mount) as is -2. `scripts/deploy_vps.sh` — the script per this spec (`set -euo pipefail`; remote steps as small quoted heredoc scripts, no unvalidated interpolation into remote shells) -3. `package.json` — `"vps:deploy": "./scripts/deploy_vps.sh"` and `"vps:env": "./scripts/deploy_vps.sh env"` +2. `scripts/vps-deploy.sh` — the script per this spec (`set -euo pipefail`; remote steps as small quoted heredoc scripts, no unvalidated interpolation into remote shells) +3. `package.json` — `"vps:deploy": "./scripts/vps-deploy.sh"` and `"vps:env": "./scripts/vps-deploy.sh env"` 4. `README.md` — rewrite Deploy to a VPS around the script; keep the manual compose flow as a short "doing it by hand" note 5. `.env.example` — update the deployment-block example values to the `editable-` container naming diff --git a/docker-compose.yml b/docker-compose.yml index b1510ee8..06aade0d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ # don't use this file (fly deploy builds the same Dockerfile directly). # # Two ways to use it: -# - scripts/deploy_vps.sh uploads this file and starts a locally built image +# - scripts/vps-deploy.sh uploads this file and starts a locally built image # (IMAGE_TAG and CONTAINER_NAME come from the .deploy_env file it writes) # - by hand on the server: `docker compose up -d --build` builds from source # and runs under the default name `editable` diff --git a/package.json b/package.json index 985bc990..e8455759 100644 --- a/package.json +++ b/package.json @@ -29,9 +29,9 @@ "data:cloud-snapshots": "./scripts/data.sh cloud-snapshots", "data:verify": "./scripts/data.sh verify", "data:reset": "./scripts/data.sh reset", - "vps:deploy": "./scripts/deploy_vps.sh", - "vps:env": "./scripts/deploy_vps.sh env", - "vps:status": "./scripts/deploy_vps.sh status", + "vps:deploy": "./scripts/vps-deploy.sh", + "vps:env": "./scripts/vps-deploy.sh env", + "vps:status": "./scripts/vps-deploy.sh status", "litestream:install": "./scripts/install-litestream.sh" }, "devDependencies": { diff --git a/scripts/data.sh b/scripts/data.sh index a38aaa9f..9bbf7676 100755 --- a/scripts/data.sh +++ b/scripts/data.sh @@ -37,7 +37,7 @@ # fly CLI. Override with -a or the FLY_APP environment variable (-a wins). # # Ssh driver: configured via environment variables or .env (environment wins). -# For a server managed by scripts/deploy_vps.sh, DEPLOY_HOST is the only key +# For a server managed by scripts/vps-deploy.sh, DEPLOY_HOST is the only key # needed — the rest is discovered from the server. The explicit keys are for # other setups (bare node, hand-managed compose) and always override: # DEPLOY_HOST user@host to ssh into (setting this selects the driver) @@ -110,7 +110,7 @@ DEPLOY_HOST="${DEPLOY_HOST:-}" REMOTE_APP_DIR="${REMOTE_APP_DIR:-/app}" REMOTE_DATA="${REMOTE_DATA_DIR:-/data}" -# A deploy_vps.sh-managed server needs only DEPLOY_HOST — container name and +# A vps-deploy.sh-managed server needs only DEPLOY_HOST — container name and # data path are discovered from its /srv//.deploy_env marker, the way # fly.toml resolves the rest for the fly driver. Explicit values always win, # and without exactly one marker (bare node, hand-managed compose, multiple diff --git a/scripts/deploy_vps.sh b/scripts/vps-deploy.sh similarity index 95% rename from scripts/deploy_vps.sh rename to scripts/vps-deploy.sh index 03d5b4a8..00b71c90 100755 --- a/scripts/deploy_vps.sh +++ b/scripts/vps-deploy.sh @@ -10,19 +10,19 @@ # container is disposable and replaced on every deploy. # # Usage -# ./scripts/deploy_vps.sh first deploy (or explicit target) -# ./scripts/deploy_vps.sh deploy to DEPLOY_HOST -# ./scripts/deploy_vps.sh status show the running tag and rollback candidates -# ./scripts/deploy_vps.sh env show the server's env (secrets masked) -# ./scripts/deploy_vps.sh env set KEY=VALUE … set env vars and restart the app -# ./scripts/deploy_vps.sh env set KEY prompt for the value (hidden input) -# ./scripts/deploy_vps.sh env unset KEY … remove env vars and restart the app +# ./scripts/vps-deploy.sh first deploy (or explicit target) +# ./scripts/vps-deploy.sh deploy to DEPLOY_HOST +# ./scripts/vps-deploy.sh status show the running tag and rollback candidates +# ./scripts/vps-deploy.sh env show the server's env (secrets masked) +# ./scripts/vps-deploy.sh env set KEY=VALUE … set env vars and restart the app +# ./scripts/vps-deploy.sh env set KEY prompt for the value (hidden input) +# ./scripts/vps-deploy.sh env unset KEY … remove env vars and restart the app # # The short forms read DEPLOY_HOST from your local .env — the deployment # block printed after the first deploy, added by hand so it's transparent # where the target comes from — and discover the site on the server. The # explicit form works before that block exists and always overrides it: -# ./scripts/deploy_vps.sh root@203.0.113.10 my-site.example.com [env …] +# ./scripts/vps-deploy.sh root@203.0.113.10 my-site.example.com [env …] # # Options # --tag deploy an already-uploaded image tag (rollback) instead of @@ -83,7 +83,7 @@ case "${POSITIONAL[0]:-}" in # Explicit form: [env …] TARGET="${POSITIONAL[0]}" DOMAIN="${POSITIONAL[1]:-}" - [ -n "$DOMAIN" ] || die "the explicit form needs a domain: deploy_vps.sh $TARGET " + [ -n "$DOMAIN" ] || die "the explicit form needs a domain: vps-deploy.sh $TARGET " ACTION="${POSITIONAL[2]:-deploy}" ENV_ARGS=("${POSITIONAL[@]:3}") case "$ACTION" in @@ -97,7 +97,7 @@ case "${POSITIONAL[0]:-}" in ACTION="${POSITIONAL[0]:-deploy}" ENV_ARGS=("${POSITIONAL[@]:1}") TARGET="${DEPLOY_HOST:-$(local_env_val DEPLOY_HOST)}" - [ -n "$TARGET" ] || die "DEPLOY_HOST is not set — add the deployment block to .env (printed after the first deploy), or pass an explicit target: deploy_vps.sh user@host domain" + [ -n "$TARGET" ] || die "DEPLOY_HOST is not set — add the deployment block to .env (printed after the first deploy), or pass an explicit target: vps-deploy.sh user@host domain" case "$TARGET" in *@*) ;; *) die "DEPLOY_HOST must be user@host, e.g. root@203.0.113.10" ;; @@ -181,9 +181,9 @@ if [ -z "$DOMAIN" ]; then DEPLOYED="$(rssh_retry 'ls /srv/*/.deploy_env 2>/dev/null' || true)" DEPLOYED_COUNT="$(printf '%s' "$DEPLOYED" | grep -c . || true)" [ "$DEPLOYED_COUNT" -ge 1 ] || - die "no Editable deployment found on $TARGET — run the first deploy explicitly: deploy_vps.sh $TARGET " + die "no Editable deployment found on $TARGET — run the first deploy explicitly: vps-deploy.sh $TARGET " [ "$DEPLOYED_COUNT" -eq 1 ] || - die "multiple sites found on $TARGET — address one explicitly: deploy_vps.sh $TARGET " + die "multiple sites found on $TARGET — address one explicitly: vps-deploy.sh $TARGET " DISCOVERED_SITE="$(basename "$(dirname "$DEPLOYED")")" echo "$DISCOVERED_SITE" | grep -Eq '^[a-z0-9]([a-z0-9-]*[a-z0-9])?$' || die "unexpected site directory name '/srv/$DISCOVERED_SITE'" From 0760ba713b9bdac9c5dbbe6722ca06183dd51897 Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Tue, 21 Jul 2026 20:33:34 +0200 Subject: [PATCH 10/13] No longer derive an app name assuming there's one app per VPS --- .env.example | 2 +- README.md | 4 ++-- VPS_DEPLOY_SPEC.md | 21 +++++++++++--------- docker-compose.yml | 5 ++++- scripts/data.sh | 18 ++++++++--------- scripts/vps-deploy.sh | 46 ++++++++++++++++++++----------------------- 6 files changed, 49 insertions(+), 47 deletions(-) diff --git a/.env.example b/.env.example index 85e61284..3297ebcf 100644 --- a/.env.example +++ b/.env.example @@ -17,4 +17,4 @@ ADMIN_PASSWORD="your-secure-password-here" # nothing to discover), or as overrides: # RESTART_CMD="docker restart editable" # REMOTE_EXEC="docker exec editable" -# HOST_DATA_DIR="/srv/my-site/data" +# HOST_DATA_DIR="/path/to/checkout/data" diff --git a/README.md b/README.md index d9a39c3e..4dd28c27 100644 --- a/README.md +++ b/README.md @@ -700,7 +700,7 @@ Editable runs on any amd64 host with Docker — a DigitalOcean droplet, a Hetzne npm run vps:deploy -- root@203.0.113.10 my-site.example.com ``` -The first run provisions the server — Docker, [Caddy](https://caddyserver.com) as the TLS-terminating reverse proxy, swap on machines with less than 2 GB RAM — asks you for an admin password, builds the Docker image locally, streams it over ssh, and starts the site. The server never needs access to your git repository or the memory to run a build. Your content lives in `/srv/my-site/data` on the server; deploys replace the container and never touch that folder. +The first run provisions the server — Docker, [Caddy](https://caddyserver.com) as the TLS-terminating reverse proxy, swap on machines with less than 2 GB RAM — asks you for an admin password, builds the Docker image locally, streams it over ssh, and starts the site. The server never needs access to your git repository or the memory to run a build. Your content lives in `/data` on the server — the same path Fly.io mounts and the same path inside the container; deploys replace the container and never touch that folder. After the first deploy, add one line to your **local** `.env` — this is your checkout's deployment identity, playing the role `fly.toml` plays for Fly.io. Everything else (container name, data path) is read from the server: @@ -753,7 +753,7 @@ What Fly.io still does for you that a VPS doesn't: scale-to-zero with sub-second Your whole site lives in one folder — pull it, push it, snapshot it, roll it back. -That folder is `data/`: an SQLite database (`db.sqlite3`) and uploaded assets (`assets/`). Locally it defaults to `./data`; on Fly.io it's a persistent volume at `/data`; on a VPS it's the `./data` bind mount next to the compose file. The data commands move that folder between your machine, your deployment, and — optionally — a backup bucket. The complete toolbox: +That folder is `data/`: an SQLite database (`db.sqlite3`) and uploaded assets (`assets/`). Locally it defaults to `./data`; on Fly.io it's a persistent volume at `/data`; on a VPS it's `/data` on the host, bind-mounted into the container at the same path. The data commands move that folder between your machine, your deployment, and — optionally — a backup bucket. The complete toolbox: - **npm run data:pull** — Copy the live site's data to your machine - **npm run data:push [-- --yes]** — Replace the live site's data with your local state — guarded, undoable diff --git a/VPS_DEPLOY_SPEC.md b/VPS_DEPLOY_SPEC.md index 814d9be7..e86e2e42 100644 --- a/VPS_DEPLOY_SPEC.md +++ b/VPS_DEPLOY_SPEC.md @@ -27,16 +27,19 @@ This file is the working spec for the feature. Once implemented and stable, fold **Compose stays the runtime, switched from `build:` to `image:`.** `docker-compose.yml` gains `image: 'editable:${IMAGE_TAG:-local}'` and drops `build: .` as the on-server path. Local from-source runs use `docker compose up --build` explicitly (compose builds the `image:` tag when `--build` is passed), so the local workflow documented today keeps working. The script deploys with `IMAGE_TAG= docker compose up -d --remove-orphans`; compose sees the tag change and replaces the container, leaving the `./data` bind mount untouched. -**Server layout.** Everything lives in one directory, `/srv/` (site name = the domain's first label, e.g. `my-site`): +**Server layout.** One site per server, so nothing needs a per-site name — the layout separates the precious from the disposable: ``` -/srv/my-site/ +/data the persistent site data — the only thing worth backing + up; same path as Fly's volume mount and the in-container + path (bind mount /data → /data) +/srv/editable/ ├── docker-compose.yml uploaded by the script on every deploy ├── .env created on first run, never overwritten -└── data/ the persistent site data (bind mount → /data) +└── .deploy_env marker: IMAGE_TAG, CONTAINER_NAME, HOST_DATA_DIR ``` -`/etc/caddy/Caddyfile` holds the reverse-proxy config. +Everything under `/srv/editable` is recreatable by the next deploy. The container is plain `editable` — identical to a hand-managed compose setup, so both flows converge on one naming scheme. The Caddy vhost lives in `/etc/caddy/sites/editable.caddy`, imported from the main Caddyfile (which stays untouched otherwise). The compose volume line is `'${HOST_DATA_DIR:-./data}:/data'`: local and hand-managed runs keep `./data` next to the compose file, the script pins `/data` via `.deploy_env`. **Secrets.** On first run the script prompts for `ADMIN_PASSWORD` (offering a generated one), sets `ORIGIN=https://`, and writes the server's `.env`. Backup-bucket credentials (`BUCKET_NAME`, `AWS_*`) are copied from the local `.env` if present. That first-run bootstrap is the only implicit sync: afterwards the server's `.env` is authoritative and changes only through the explicit `env` command (`env` show / `env set KEY=VALUE` / `env set KEY` with hidden prompt / `env unset KEY`), which rewrites the file and recreates the container so the change takes effect — fly-secrets style, nothing moves without being named. Secrets are never passed as command-line arguments to remote shells. @@ -66,17 +69,17 @@ Options: --yes skip confirmation prompts (except the first-run password prompt) ``` -**Addressing.** `DEPLOY_HOST` in the local `.env` is the checkout's entire deployment identity — the same key the data toolbox uses, playing the role `fly.toml` plays for Fly.io. The user adds it by hand from the line the first deploy prints (deliberately not auto-written, so it stays transparent where commands are targeted). Everything else is discovered on the server via the `/srv//.deploy_env` marker (exactly one per the one-site-per-server rule): the deploy script reads the site and its domain (from `ORIGIN` in the server's `.env`), and `data.sh` derives `REMOTE_EXEC`, `RESTART_CMD`, and `HOST_DATA_DIR` from the marker's `CONTAINER_NAME` and path. Explicit values always override, and setups without the marker (bare node, hand-managed compose) keep configuring the data toolbox explicitly. The explicit ` ` form is what works before any of this exists. +**Addressing.** `DEPLOY_HOST` in the local `.env` is the checkout's entire deployment identity — the same key the data toolbox uses, playing the role `fly.toml` plays for Fly.io. The user adds it by hand from the line the first deploy prints (deliberately not auto-written, so it stays transparent where commands are targeted). Everything else is read from the server via the `/srv/editable/.deploy_env` marker: the deploy script reads the domain from `ORIGIN` in the server's `.env`, and `data.sh` takes `REMOTE_EXEC`, `RESTART_CMD`, and `HOST_DATA_DIR` from the marker's `CONTAINER_NAME` and `HOST_DATA_DIR`. Explicit values always override, and setups without the marker (bare node, hand-managed compose) keep configuring the data toolbox explicitly. The explicit ` ` form is what works before any of this exists. `` is typically `root@` on a fresh droplet; any sudo-capable user works. ssh key access is assumed (the script never handles passwords). ## Execution phases 1. **Preflight (local).** Verify: git worktree present, `docker buildx` available, ssh connectivity to the host (`BatchMode=yes`), remote architecture is x86_64 (abort otherwise), domain resolves to the host's IP (warn, don't abort — DNS may still be propagating). -2. **Provision (remote, idempotent).** Install Docker (official convenience script) and Caddy (apt repo) if missing; create 1 GB swap if total RAM < 2 GB and no swap exists; create `/srv//data`. +2. **Provision (remote, idempotent).** Install Docker (official convenience script) and Caddy (apt repo) if missing; create 1 GB swap if total RAM < 2 GB and no swap exists; create `/data` and `/srv/editable`. 3. **Configure (remote, idempotent).** Write `/etc/caddy/Caddyfile` (reverse_proxy block for the domain) and reload Caddy if it changed. First run: prompt for `ADMIN_PASSWORD`, write `.env` with it plus `ORIGIN` and any local bucket credentials. 4. **Build & upload (local → remote).** Skipped with `--tag`. Build `editable:` for linux/amd64, stream via `ssh docker load`. Upload `docker-compose.yml`. -5. **Activate (remote).** `IMAGE_TAG= docker compose up -d --remove-orphans` in `/srv/` (tag passed via a `.deploy_tag` env file, not shell interpolation). +5. **Activate (remote).** `IMAGE_TAG= docker compose up -d --remove-orphans` in `/srv/editable` (tag passed via the `.deploy_env` file, not shell interpolation). 6. **Verify.** Poll `127.0.0.1:3000` via ssh until healthy or timeout; on success also curl `https://` from the local machine (warn-only — TLS issuance or DNS may lag). Print logs and fail otherwise. 7. **Cleanup & report.** Prune `editable:*` images beyond the newest 3. Print the deployed tag, the site URL, and (first run) the local `.env` block for the data commands. @@ -86,7 +89,7 @@ Options: 2. `scripts/vps-deploy.sh` — the script per this spec (`set -euo pipefail`; remote steps as small quoted heredoc scripts, no unvalidated interpolation into remote shells) 3. `package.json` — `"vps:deploy": "./scripts/vps-deploy.sh"` and `"vps:env": "./scripts/vps-deploy.sh env"` 4. `README.md` — rewrite Deploy to a VPS around the script; keep the manual compose flow as a short "doing it by hand" note -5. `.env.example` — update the deployment-block example values to the `editable-` container naming +5. `.env.example` — `DEPLOY_HOST` as the only needed key, explicit keys as hand-managed overrides ## Implementation steps @@ -104,4 +107,4 @@ Status: implemented (compose switch, script, `vps:deploy` / `vps:env` / `vps:sta ## Open questions - Should the script optionally harden the box (ufw allowing 22/80/443, unattended-upgrades)? Writebook's installer does some of this. Leaning yes for ufw, no for anything beyond — but deferred until after v1 works end to end. -- One site per server is the decided v1 scope. Multi-site later is additive, not a redesign — the collisions are: the fixed host port 3000, the hardcoded `container_name: editable`, the script owning all of `/etc/caddy/Caddyfile`, and the shared `editable:` image namespace. To keep that door open cheaply, v1 adopts two conventions now: name the container `editable-`, and write the Caddy config as `/etc/caddy/sites/.caddy` imported from the main Caddyfile. A future multi-site step then only needs a per-site `APP_PORT` and per-site image repos (`editable-:`). +- Multi-site per server was considered and decided against: the tool stays committed to one site per server, hardened and simplified, rather than growing toward a mini-PaaS. Consequence: nothing is scoped by a site name — fixed paths (`/data`, `/srv/editable`), fixed container name (`editable`), no name derivation. Anyone wanting many sites runs many cheap servers (or uses Fly.io). diff --git a/docker-compose.yml b/docker-compose.yml index 06aade0d..433eae6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,4 +26,7 @@ services: DATA_DIR: /data BODY_SIZE_LIMIT: '${BODY_SIZE_LIMIT:-30000000}' volumes: - - ./data:/data + # Host data location: ./data next to this file by default (local and + # hand-managed runs); vps-deploy.sh pins it to /data via .deploy_env so + # the host path matches the container path and Fly's volume mount. + - '${HOST_DATA_DIR:-./data}:/data' diff --git a/scripts/data.sh b/scripts/data.sh index 9bbf7676..aa652489 100755 --- a/scripts/data.sh +++ b/scripts/data.sh @@ -110,21 +110,21 @@ DEPLOY_HOST="${DEPLOY_HOST:-}" REMOTE_APP_DIR="${REMOTE_APP_DIR:-/app}" REMOTE_DATA="${REMOTE_DATA_DIR:-/data}" -# A vps-deploy.sh-managed server needs only DEPLOY_HOST — container name and -# data path are discovered from its /srv//.deploy_env marker, the way +# A vps-deploy.sh-managed server needs only DEPLOY_HOST — the marker file at +# /srv/editable/.deploy_env names the container and host data path, the way # fly.toml resolves the rest for the fly driver. Explicit values always win, -# and without exactly one marker (bare node, hand-managed compose, multiple -# sites) nothing changes and the explicit keys below stay required. +# and without the marker (bare node, hand-managed compose) nothing changes +# and the explicit keys below stay required. if [ "$DRIVER" = "ssh" ] && [ -n "$DEPLOY_HOST" ] && [ -z "${REMOTE_EXEC:-}" ] && [ -z "${RESTART_CMD:-}" ] && [ -z "${HOST_DATA_DIR:-}" ]; then - FOUND="$(ssh "$DEPLOY_HOST" 'for f in /srv/*/.deploy_env; do [ -f "$f" ] && printf "%s %s\n" "$f" "$(sed -n s/^CONTAINER_NAME=//p "$f")"; done' 2>/dev/null || true)" - if [ "$(printf '%s' "$FOUND" | grep -c .)" -eq 1 ]; then - MARKER="${FOUND%% *}" - CONTAINER="${FOUND#* }" + MARKER="$(ssh "$DEPLOY_HOST" 'cat /srv/editable/.deploy_env 2>/dev/null' 2>/dev/null || true)" + if [ -n "$MARKER" ]; then + CONTAINER="$(printf '%s\n' "$MARKER" | sed -n 's/^CONTAINER_NAME=//p')" if printf '%s' "$CONTAINER" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9._-]*$'; then REMOTE_EXEC="docker exec $CONTAINER" RESTART_CMD="docker restart $CONTAINER" - HOST_DATA_DIR="$(dirname "$MARKER")/data" + HOST_DATA_DIR="$(printf '%s\n' "$MARKER" | sed -n 's/^HOST_DATA_DIR=//p')" + HOST_DATA_DIR="${HOST_DATA_DIR:-/data}" fi fi fi diff --git a/scripts/vps-deploy.sh b/scripts/vps-deploy.sh index 00b71c90..ab9a9fe4 100755 --- a/scripts/vps-deploy.sh +++ b/scripts/vps-deploy.sh @@ -6,8 +6,8 @@ # TLS-terminated site; the same command run again ships a code update. The # image is built locally and streamed over ssh — the server never needs git # access, npm, or the memory to run a build. The only state that matters on -# the server is the bind-mounted data directory (/srv//data); the -# container is disposable and replaced on every deploy. +# the server is the bind-mounted data directory /data (the same path Fly.io +# mounts); the container is disposable and replaced on every deploy. # # Usage # ./scripts/vps-deploy.sh first deploy (or explicit target) @@ -109,15 +109,19 @@ esac # Values interpolated into remote commands are validated to safe character # sets first — everything else reaches the remote side via stdin only. -derive_site() { +validate_domain() { DOMAIN="$(echo "$DOMAIN" | tr '[:upper:]' '[:lower:]')" echo "$DOMAIN" | grep -Eq '^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$' || die "'$DOMAIN' does not look like a domain name" - SITE="${DOMAIN%%.*}" - SRV="/srv/$SITE" - CONTAINER="editable-$SITE" } -[ -z "$DOMAIN" ] || derive_site +[ -z "$DOMAIN" ] || validate_domain + +# One site per server, so nothing needs a per-site name: runtime files live +# in /srv/editable, the data in /data (the same path Fly.io mounts and the +# same path inside the container), and the container is plain 'editable' — +# identical to a hand-managed compose setup. +SRV="/srv/editable" +CONTAINER="editable" if [ -n "$TAG" ]; then echo "$TAG" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9._-]*$' || die "'$TAG' is not a valid image tag" @@ -176,21 +180,13 @@ fi info "Checking ssh connectivity to $TARGET" rssh_retry true 2>/dev/null || die "cannot ssh into $TARGET (key-based access required)" -# Short form: discover the (single) site on the server and its domain. +# Short form: read the deployed site's domain from the server. if [ -z "$DOMAIN" ]; then - DEPLOYED="$(rssh_retry 'ls /srv/*/.deploy_env 2>/dev/null' || true)" - DEPLOYED_COUNT="$(printf '%s' "$DEPLOYED" | grep -c . || true)" - [ "$DEPLOYED_COUNT" -ge 1 ] || + rssh_retry "test -f $SRV/.deploy_env" 2>/dev/null || die "no Editable deployment found on $TARGET — run the first deploy explicitly: vps-deploy.sh $TARGET " - [ "$DEPLOYED_COUNT" -eq 1 ] || - die "multiple sites found on $TARGET — address one explicitly: vps-deploy.sh $TARGET " - DISCOVERED_SITE="$(basename "$(dirname "$DEPLOYED")")" - echo "$DISCOVERED_SITE" | grep -Eq '^[a-z0-9]([a-z0-9-]*[a-z0-9])?$' || - die "unexpected site directory name '/srv/$DISCOVERED_SITE'" - DOMAIN="$(rssh_retry "$SUDO cat /srv/$DISCOVERED_SITE/.env" | sed -n 's|^ORIGIN="https://\([^"]*\)".*|\1|p' | sed -n '1p')" - [ -n "$DOMAIN" ] || die "could not read the site's domain (ORIGIN) from /srv/$DISCOVERED_SITE/.env" - derive_site - [ "$SITE" = "$DISCOVERED_SITE" ] || die "site directory /srv/$DISCOVERED_SITE does not match its ORIGIN domain $DOMAIN" + DOMAIN="$(rssh_retry "$SUDO cat $SRV/.env" | sed -n 's|^ORIGIN="https://\([^"]*\)".*|\1|p' | sed -n '1p')" + [ -n "$DOMAIN" ] || die "could not read the site's domain (ORIGIN) from $SRV/.env" + validate_domain info "Site: $DOMAIN ($TARGET)" fi @@ -315,7 +311,7 @@ fi info "Provisioning server (no-op when already set up)" { - printf 'SITE=%q\nAPP_USER=%q\n' "$SITE" "$REMOTE_USER" + printf 'APP_USER=%q\n' "$REMOTE_USER" cat <<'REMOTE' set -euo pipefail export DEBIAN_FRONTEND=noninteractive @@ -345,8 +341,8 @@ if [ "$total_kb" -lt 2000000 ] && [ "$(swapon --noheadings | wc -l)" -eq 0 ]; th grep -q '^/swapfile' /etc/fstab || echo '/swapfile none swap sw 0 0' >>/etc/fstab fi -mkdir -p "/srv/$SITE/data" -[ "$APP_USER" = "root" ] || chown "$APP_USER" "/srv/$SITE" +mkdir -p /data /srv/editable +[ "$APP_USER" = "root" ] || chown "$APP_USER" /data /srv/editable REMOTE } | rssh "$SUDO bash -s" @@ -358,7 +354,7 @@ info "Configuring Caddy for $DOMAIN" cat <<'REMOTE' set -euo pipefail mkdir -p /etc/caddy/sites -site_file="/etc/caddy/sites/$DOMAIN.caddy" +site_file="/etc/caddy/sites/editable.caddy" desired="$DOMAIN { reverse_proxy 127.0.0.1:3000 }" @@ -430,7 +426,7 @@ rssh "$SUDO tee $SRV/docker-compose.yml >/dev/null" /dev/null" rssh_retry "cd $SRV && $SUDO docker compose --env-file .env --env-file .deploy_env up -d --remove-orphans" From 4cfeaf29f9ba99d2ee431315d45f37c6de91b0db Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Tue, 21 Jul 2026 23:17:41 +0200 Subject: [PATCH 11/13] Optimize Dockerfile --- Dockerfile | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7cb87ed7..3bc4382e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:1 ARG NODE_VERSION=24.14.0 -FROM node:${NODE_VERSION}-slim as base +FROM node:${NODE_VERSION}-slim AS base LABEL fly_launch_runtime="Node.js" @@ -9,18 +9,28 @@ WORKDIR /app ENV NODE_ENV="production" -# Build stage -FROM base as build +# Install dependencies before copying application code so this layer stays +# reusable until package.json or package-lock.json changes. +FROM base AS dependencies COPY --link .npmrc package-lock.json package.json ./ RUN npm ci --include=dev -COPY --link . . - -RUN mkdir /data && npm run build +# Produce a runtime-only dependency tree from the same clean install. This +# stage deliberately does not depend on application source, so code-only +# changes cannot invalidate the final image's node_modules layer. +FROM dependencies AS production-dependencies RUN npm prune --omit=dev +# Build stage +FROM dependencies AS build + +COPY --link . . + +RUN mkdir /data && npm run build && \ + mv /app/node_modules /build-dependencies + # Final stage FROM base @@ -33,14 +43,19 @@ ARG LITESTREAM_VERSION=0.5.14 ADD https://github.com/benbjohnson/litestream/releases/download/v${LITESTREAM_VERSION}/litestream-${LITESTREAM_VERSION}-linux-x86_64.deb /tmp/litestream.deb RUN dpkg -i /tmp/litestream.deb && rm /tmp/litestream.deb -COPY --from=build /app /app +# Keep the large dependency tree separate from the frequently changing +# application so registries and Docker's local image store can share it across +# code-only releases. The build stage moved its development dependencies out +# of /app, making the broad application copy below safe and omission-proof. +COPY --link --from=production-dependencies /app/node_modules /app/node_modules +COPY --link --from=build /app /app # Copy .sqliterc for convenient sqlite3 CLI usage -COPY --from=build /app/.sqliterc /root/.sqliterc +COPY --link --from=build /app/.sqliterc /root/.sqliterc RUN mkdir -p /data VOLUME /data EXPOSE 3000 -CMD ["node", "/app/scripts/start-app.js"] \ No newline at end of file +CMD ["node", "/app/scripts/start-app.js"] From 2b9361bdc2bfed0a78a7a05afb41815c76737a8d Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Tue, 21 Jul 2026 23:24:36 +0200 Subject: [PATCH 12/13] Make sure only referenced assets are pushed from local to remote --- README.md | 2 ++ scripts/check-assets.js | 33 ++++++++++++++++++++++++++++----- scripts/data.sh | 12 ++++++++---- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4dd28c27..7c5ea7b0 100644 --- a/README.md +++ b/README.md @@ -669,6 +669,8 @@ fly secrets set \ ADMIN_PASSWORD='pick-a-strong-password' ``` +`ORIGIN` must exactly match the URL you use in the browser, including the scheme and subdomain (for example, `https://example.com` and `https://www.example.com` are different origins). An incorrect value causes login and other write requests to fail with `403 Forbidden` before the password is checked. Update this secret whenever you switch to a custom domain, and access the site through that canonical URL. + Optionally set `ASSET_GRACE_PERIOD_DAYS` (default 7): unreferenced asset files are kept on disk this many days after losing their last reference. This is also the safe window for rolling back a database backup against the live assets folder without ending up with dead image references. Deploy. The first deploy also creates the 1 GB `data` volume declared under `[mounts]` in `fly.toml`: diff --git a/scripts/check-assets.js b/scripts/check-assets.js index 4be28f18..2abe42e4 100644 --- a/scripts/check-assets.js +++ b/scripts/check-assets.js @@ -4,17 +4,19 @@ // and after a push (in-container, against the live volume). Standalone: no // $lib imports, so it runs against the built image too. // -// Usage: node --disable-warning=ExperimentalWarning check-assets.js +// Usage: node --disable-warning=ExperimentalWarning check-assets.js [--list-entries] // Exit codes: 0 = all present, 1 = missing references, 2 = bad usage. import { DatabaseSync } from 'node:sqlite'; -import { existsSync } from 'node:fs'; -import { join } from 'node:path'; +import { existsSync, lstatSync } from 'node:fs'; +import { extname, join } from 'node:path'; -const [, , db_path, assets_dir] = process.argv; +const args = process.argv.slice(2); +const list_entries = args[0] === '--list-entries'; +const [db_path, assets_dir] = list_entries ? args.slice(1) : args; if (!db_path || !assets_dir) { - console.error('usage: check-assets.js '); + console.error('usage: check-assets.js [--list-entries] '); process.exit(2); } @@ -48,5 +50,26 @@ if (missing.length > 0) { process.exit(1); } +if (list_entries) { + // `referenced` comes exclusively from the database above. Responsive + // variants are not stored as separate database references: the application + // derives their URLs and storage directory from the referenced original's + // hash. Include that directory when it exists, following the same storage + // convention as asset_storage.js. + const entries = new Set(referenced); + for (const asset_id of referenced) { + const ext = extname(asset_id); + if (!ext) continue; + const asset_stem = asset_id.slice(0, -ext.length); + try { + if (lstatSync(join(assets_dir, asset_stem)).isDirectory()) entries.add(asset_stem); + } catch { + // This referenced original has no responsive variants on disk. + } + } + for (const entry of [...entries].sort()) console.log(entry); + process.exit(0); +} + // Parsed by remote-db.sh (summary) and data.sh (verify) — keep the shape. console.log(`OK: all ${plural(referenced.size, 'referenced asset')} present`); diff --git a/scripts/data.sh b/scripts/data.sh index aa652489..58eabbf0 100755 --- a/scripts/data.sh +++ b/scripts/data.sh @@ -295,7 +295,11 @@ cmd_push() { remote prune-backups "$KEEP_BACKUPS" info "Syncing assets (additive)…" - list_local_assets >"$TMP/local_assets" + # Push only the snapshot's working set (referenced originals and their + # variant directories). Unreferenced local history must not be copied to a + # deployment merely because it is still inside the local grace period. + node --disable-warning=ExperimentalWarning "$SCRIPT_DIR/check-assets.js" \ + --list-entries "$TMP/push.db" "$DATA_DIR_LOCAL/assets" | sort >"$TMP/local_assets" # The listing must provably succeed — a failed connection must not read # as an empty asset list (pull would silently miss media). list-assets # prints a '#' header, so success is never empty even with zero assets. @@ -305,9 +309,9 @@ cmd_push() { comm -23 "$TMP/local_assets" "$TMP/remote_assets" >"$TMP/to_push" || true if [ -s "$TMP/to_push" ]; then info " $(plural "$(wc -l <"$TMP/to_push" | tr -d ' ')" 'new asset entry' 'new asset entries')" - # COPYFILE_DISABLE: keep macOS tar from embedding xattr headers that - # GNU tar on the server warns about. - COPYFILE_DISABLE=1 tar -czf "$TMP/assets.tgz" -C "$DATA_DIR_LOCAL/assets" --exclude '.DS_Store' -T "$TMP/to_push" + # Keep macOS tar from embedding xattr headers that GNU tar on the server + # warns about. COPYFILE_DISABLE also prevents AppleDouble sidecars. + COPYFILE_DISABLE=1 tar --no-xattrs -czf "$TMP/assets.tgz" -C "$DATA_DIR_LOCAL/assets" --exclude '.DS_Store' -T "$TMP/to_push" sftp_put "$TMP/assets.tgz" "$REMOTE_DATA/incoming/assets.tgz" remote extract-assets else From 6a44ab37c9216a3c2d84dc8507764685efec55eb Mon Sep 17 00:00:00 2001 From: Michael Aufreiter Date: Tue, 21 Jul 2026 23:38:13 +0200 Subject: [PATCH 13/13] Fix a problem where npm run data:help hung when DEPLOY_HOST is present --- scripts/data.sh | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/scripts/data.sh b/scripts/data.sh index 58eabbf0..536a6c98 100755 --- a/scripts/data.sh +++ b/scripts/data.sh @@ -109,28 +109,35 @@ fi DEPLOY_HOST="${DEPLOY_HOST:-}" REMOTE_APP_DIR="${REMOTE_APP_DIR:-/app}" REMOTE_DATA="${REMOTE_DATA_DIR:-/data}" +HOST_DATA_DIR="${HOST_DATA_DIR:-}" +REMOTE_EXEC="${REMOTE_EXEC:-}" # A vps-deploy.sh-managed server needs only DEPLOY_HOST — the marker file at # /srv/editable/.deploy_env names the container and host data path, the way # fly.toml resolves the rest for the fly driver. Explicit values always win, # and without the marker (bare node, hand-managed compose) nothing changes # and the explicit keys below stay required. -if [ "$DRIVER" = "ssh" ] && [ -n "$DEPLOY_HOST" ] && - [ -z "${REMOTE_EXEC:-}" ] && [ -z "${RESTART_CMD:-}" ] && [ -z "${HOST_DATA_DIR:-}" ]; then - MARKER="$(ssh "$DEPLOY_HOST" 'cat /srv/editable/.deploy_env 2>/dev/null' 2>/dev/null || true)" - if [ -n "$MARKER" ]; then - CONTAINER="$(printf '%s\n' "$MARKER" | sed -n 's/^CONTAINER_NAME=//p')" - if printf '%s' "$CONTAINER" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9._-]*$'; then - REMOTE_EXEC="docker exec $CONTAINER" - RESTART_CMD="docker restart $CONTAINER" - HOST_DATA_DIR="$(printf '%s\n' "$MARKER" | sed -n 's/^HOST_DATA_DIR=//p')" - HOST_DATA_DIR="${HOST_DATA_DIR:-/data}" +SSH_CONFIG_DISCOVERED=false +discover_ssh_config() { + [ "$SSH_CONFIG_DISCOVERED" = false ] || return + SSH_CONFIG_DISCOVERED=true + + if [ "$DRIVER" = "ssh" ] && [ -n "$DEPLOY_HOST" ] && + [ -z "$REMOTE_EXEC" ] && [ -z "${RESTART_CMD:-}" ] && [ -z "$HOST_DATA_DIR" ]; then + local marker container + marker="$(ssh "$DEPLOY_HOST" 'cat /srv/editable/.deploy_env 2>/dev/null' 2>/dev/null || true)" + if [ -n "$marker" ]; then + container="$(printf '%s\n' "$marker" | sed -n 's/^CONTAINER_NAME=//p')" + if printf '%s' "$container" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9._-]*$'; then + REMOTE_EXEC="docker exec $container" + RESTART_CMD="docker restart $container" + HOST_DATA_DIR="$(printf '%s\n' "$marker" | sed -n 's/^HOST_DATA_DIR=//p')" + fi fi fi -fi -HOST_DATA_DIR="${HOST_DATA_DIR:-$REMOTE_DATA}" -REMOTE_EXEC="${REMOTE_EXEC:-}" + HOST_DATA_DIR="${HOST_DATA_DIR:-$REMOTE_DATA}" +} if [ "$DRIVER" = "ssh" ]; then APP="${DEPLOY_NAME:-${DEPLOY_HOST#*@}}" @@ -146,6 +153,7 @@ info() { echo "→ $*"; } plural() { [ "$1" -eq 1 ] && echo "$1 $2" || echo "$1 ${3:-${2}s}"; } need_app() { + discover_ssh_config if [ "$DRIVER" = "ssh" ]; then [ -n "${DEPLOY_HOST:-}" ] || die "No deploy host configured — set DEPLOY_HOST='user@host' in .env or the environment" else