diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index b6f22ca..d6eff75 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -386,6 +386,10 @@ into each rather than duplicated as new work items. - TCL-44 — F37/host-helper (constrained extractor, entry policy, bounds). - TCL-45 — scheduled/manual backup unification (one engine + artifact schema + private workspaces + protected credentials). Architectural. + Partially landed 2026-10-20 (D3): scheduled jobs run the same Client + engine as manual creates (`backup run`), artifacts share one naming + scheme, and S3 credentials travel encrypted at rest; MYSQL_PWD + transport and engine-side credential plumbing remain. - TCL-47 — engine auth adapters + S3 session tokens. Medium; with F37. - TCL-48 — F42 (durable webhook queue). - TCL-49 — F40 (pin webhook builds to the event commit): LANDED @@ -628,7 +632,10 @@ all packages ok. No push performed. fail-open-on-parse-error that F25 closed. - A33 — F22: backup credentials in host-visible command text (AWS env assignments, MYSQL_PWD via docker exec -e); needs the container-side - credential-file plumbing shared with the engine images. + credential-file plumbing shared with the engine images. Partially + improved 2026-10-20 (D3): the stored server backup target decrypts its + S3 credentials inside the invocation's own shell (never argv, never the + crontab); the legacy --endpoint flag path and MYSQL_PWD remain. - A34 — F42 durable webhook queue (ack-before-durable-job remains; A36 closed the dedup-race half). - A35 — F40 webhook build pinning to the event commit: LANDED 2026-09-22 diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a664d..c114304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,67 @@ All notable changes to teploy are documented here. Format follows [Keep a Change ### Added +- One backup target per server, read at run time (`backup-target-v1`; + contracts corpus rev 14). `teploy server backup-target set --host S + --bucket B [--region] [--endpoint] [--prefix]` stores it at + `/etc/teploy/backup-target.json` (0600, outside `/deployments`; + `~/.config/teploy/` fallback for a non-root deploy user, said so on + output), with the S3 credentials read from stdin as two lines (access + key, then secret key; prompted with echo off on a terminal) and stored + age-encrypted with the server's own identity — used whether or not an + endpoint is set, ambient AWS chain only when none are stored. + `show [--json]` never emits secrets; `unset` removes the target. + Setting a target rewrites teploy-generated backup cron lines (by marker) + to the `backup run` form — hand-written commands under a teploy marker + are left untouched and reported — and `teploy doctor` gains a + `backup-cron` check flagging cron lines with embedded credentials + (fail) or bucket flags (warn). +- `teploy backup create|list|restore|prune|schedule [--app X --host S] + --json` are folderless and default to the stored target + (`backup-json-v1`): `list --json` reports parsed rows + `{id, kind: volumes|accessory|teploy, accessory?, size_bytes, + created_at, key}` across the volumes, accessories and teploy prefixes + plus schedules discovered from teploy cron markers (was: raw + `aws s3 ls` lines); `create --json` reports the created backup's + identity and size; `restore --json` reports the quiesced receipt. The + legacy `--bucket`-style flags still work and override the stored + target. `teploy dr` and `teploy accessory backup|restore| + verify-backup` default to the stored target as well. +- `teploy backup run --app X [--accessory Y]` — the cron entrypoint, runs + on the server like `autodeploy redeploy` and reads the target and the + per-app retention at run time. Scheduled jobs are a single + argument-free cron line; retention moves to a per-app store + (`backup schedule` and `accessory backup --schedule` write it; + `--max-age-days` is new on the latter). `backup unschedule` removes a + schedule. Failure alerting moved from the cron line into `backup run` + (same `/deployments//backup-alert.sh`, same signatures). +- Restore is now quiesced: under the app's fenced lock the app's + containers (and a generic accessory's container, for accessory + restores) are stopped, the backup is promoted with the existing + two-phase recovery discipline, the containers are started again and + health-checked. On failure — or an unhealthy app afterwards — the + pre-restore recovery copy is kept and named. Non-interactive restores + require `--yes`. +- Events `backup.run`, `backup.restore` and `backup_target.set` (detail + allowlist: id, kind, accessory, error — never bucket or credential + material; the server-level target event is fanned out to every app + known on the server). + +### Changed + +- `backup list --json` now emits the versioned envelope above instead of + a bare array of raw `aws s3 ls` lines. The old shape was never a + contracts-corpus artifact and no consumer is known to parse it; + consumers should gate on `backup-json-v1`. + +### Deferred + +- A `backup: {schedule, keep_last, max_age_days, accessories}` app-config + key (schedule/retention from Teploy-home config) is deferred: making a + config key authoritative over crontab entries hooks cron reconciliation + into deploy, which does not fit this change. Per-app schedules remain + managed with `backup schedule --app X --host S` (folderless), which + stores both halves server-side. - Folderless per-command reads (server-authority plan D2): `teploy secret set/get/list/rm/rotate --app X --host S` read exactly the server facts they need — no teploy.yml, no reconstructed config. The secrets provider @@ -48,7 +109,6 @@ All notable changes to teploy are documented here. Format follows [Keep a Change old path so an older teploy keeps encrypting to the same identity; `teploy doctor` reports the location (`age-identity` check). Non-root deploy users that cannot write `/etc/teploy` keep using the old path. - ## [0.1.44] - 2026-10-19 ### Added diff --git a/README.md b/README.md index 09a96ba..6daff43 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ standalone. | **Asset bridging** | Shared volume prevents 404s on hashed assets during deploys | | **Destinations** | `teploy deploy -d staging` — per-environment config overlays | | **Notifications** | Webhook notifications on deploy/rollback/failure | -| **Backups** | `teploy backup` — volumes to S3 | +| **Backups** | `teploy backup` — volumes to S3; `teploy server backup-target` — the per-server target | | **Templates** | One-command deploys for common self-hosted apps | | **Deploy locking** | `teploy lock` — freeze deploys during incidents; `teploy lock status --json` reports who holds the lock | | **Vulnerability gate** | `scan: true` — Trivy scan blocks deploys on fixable CRITICALs | @@ -563,30 +563,60 @@ The mode is recorded with the preview, so redeploying the branch keeps it. `preview list --json` rows carry a `url` with the scheme actually served. ### Backups -``` -teploy backup create # backup volumes to S3 -teploy backup list / restore -teploy backup schedule # cron-driven backups -teploy backup prune # apply a retention policy -# Retention: keep only the newest N, and/or drop anything older than D days. -teploy backup create --bucket b --keep-last 7 # auto-prune after backup -teploy backup schedule "0 3 * * *" --bucket b --keep-last 7 # nightly, keep 7 -teploy backup prune --bucket b --keep-last 3 --max-age-days 30 -teploy backup prune --bucket b --accessory db --keep-last 5 -``` +One backup target per server, stored on the server and read at run time — +no bucket or credentials in crontabs, scripts or laptops: + +``` +# Store the target (credentials on stdin, two lines; age-encrypted on the +# server at /etc/teploy/backup-target.json, 0600): +printf 'ACCESS\nSECRET\n' | teploy server backup-target set --host prod \ + --bucket b [--region r] [--endpoint https://s3.example.com] [--prefix p] +teploy server backup-target show --host prod [--json] # never secrets +teploy server backup-target unset --host prod + +teploy backup create --app myapp --host prod [--json] # volumes (+ server data) +teploy backup create --app myapp --host prod --accessory db --json +teploy backup list --app myapp --host prod --json # parsed rows + schedules +teploy backup restore --app myapp --host prod --yes --json +teploy backup schedule "0 3 * * *" --app myapp --host prod --keep-last 7 +teploy backup prune --app myapp --host prod --keep-last 3 --max-age-days 30 +``` + +Every command also works from an app directory without `--app/--host`, and +the legacy `--bucket/--region/--endpoint` flags still work (they override +the stored target). `teploy dr` and `teploy accessory verify-backup` +default to the stored target too. + +Stored credentials are used whether or not an endpoint is set; without +them the server's ambient AWS credential chain applies. A non-root deploy +user who cannot write `/etc/teploy` gets `~/.config/teploy/` instead, and +the command says so. + +Scheduled jobs are a single argument-free line — +`/usr/local/bin/teploy backup run --app myapp` — with the retention half +of the schedule stored per app and read at run time, so changing either +never rewrites cron. `teploy server backup-target set` rewrites existing +teploy-generated backup cron lines to that form (hand-written commands +under a teploy marker are left alone and reported), and `teploy doctor` +flags any cron line that still embeds credentials or bucket flags. + +Restore quiesces: the app's containers (and a generic accessory's) are +stopped under the app's fenced lock, the backup is promoted with the +two-phase recovery discipline, the containers are started again and +health-checked. On failure — or an unhealthy app afterwards — the +pre-restore recovery copy is kept and named. Non-interactive restores +require `--yes`. + `--keep-last N` is a floor (the newest N are always kept, even if old); `--max-age-days D` deletes older backups but never below that floor. A failed backup never triggers a prune. Both `backup create` and scheduled cron backups alert on failure via the app's `notifications` config — `create` uses all -channels; the headless cron job posts to the first **webhook** channel (SMTP and -other channels need the CLI, which isn't on the server). - -`backup schedule` uploads a small alert script to -`/deployments//backup-alert.sh` (mode 0700) and points cron at it, rather -than inlining a `curl` in the crontab line. That is what lets the alert be -signed: the signature covers a timestamp, `date +%s` contains a `%`, and crontab -treats `%` as a newline escape. If a signing secret is configured (see +channels; the headless `backup run` posts to the first **webhook** channel +through the alert script `backup schedule` uploads to +`/deployments//backup-alert.sh` (mode 0700). That is what lets the alert +be signed: the signature covers a timestamp, `date +%s` contains a `%`, and +crontab treats `%` as a newline escape. If a signing secret is configured (see `notifications.secret` / `TEPLOY_WEBHOOK_SECRET`) the alert carries the same `X-Teploy-Signature` headers as every other delivery; otherwise it goes unsigned, and `backup schedule` says so on its output so a verifying receiver diff --git a/contracts/MANIFEST.md b/contracts/MANIFEST.md index 4002921..cf6cf46 100644 --- a/contracts/MANIFEST.md +++ b/contracts/MANIFEST.md @@ -10,8 +10,8 @@ Neutron/Nucleus dependency and a public mirror. | Corpus rev | Emitting CLI | Machine Interface | Notes | |---|---|---|---| -| 14 | main (server-authority plan D2, folderless reads) | 2 | Additive. New artifacts releases-envelope (`teploy releases --app X --host S --json`: container release records newest first with current/rollback_available (+reason); static apps list retained releases — both via the envelope, the folder-static legacy bare array is unchanged), rollback-result (`teploy rollback [--to X] --json`: {app, server, from, to, ok}; failures keep the error envelope) and redeploy-result (`teploy redeploy [--json]`: {app, server, home, release, image, ok}). Fixtures GENERATED from the real DTOs. version-handshake advertises `secret-stdin-v1` (secret set KEY --stdin; folderless secret set/get/list/rm/rotate), `releases-json-v1`, `rollback-json-v1`, `redeploy-v1` (fresh deploy of the current release's recorded image, env re-resolved) and `env-config-guard-v1` (env set/unset refuse keys a Teploy-home config declares) (regenerated mi2.json). No MI bump. | -| 13 | main (server-authority spec P2 support) | 2 | Additive. New artifact config-merge-envelope (`teploy config merge --base --file - --json`: merged config + conflicting keys, read-only), fixture GENERATED from the real configMergeDTO. version-handshake advertises `config-merge-v1`, `config-unprivileged-v1` (`config ... --unprivileged` refuses host-reaching changes) and `deploy-home-v1` (`deploy --app X --image T --home teploy` creates a new Teploy-home app) (regenerated mi2.json). No MI bump. | +| 15 | main (server-authority spec D3: backup target) | 2 | Additive. One backup target per server, stored outside /deployments with age-encrypted credentials and read at run time (`server backup-target set/show/unset`; `backup-target-v1`). New artifacts backup-target-envelope, backup-list-envelope, backup-create-envelope, backup-restore-envelope — fixtures GENERATED from the real DTO encoders (`backup list --json` now reports parsed rows {id, kind, accessory, size_bytes, created_at, key} plus schedules discovered from teploy cron markers; the old bare-array-of-raw-`aws s3 ls`-lines `backup list --json` shape is gone — it was never a corpus artifact and pinned no consumer). `backup create/restore --json` envelopes; restore quiesces the app under its fenced lock (backup-json-v1). version-handshake advertises `backup-target-v1` and `backup-json-v1` (regenerated mi2.json). No MI bump. | +| 14 | main (server-authority plan D2, folderless reads) | 2 | Additive. New artifacts releases-envelope (`teploy releases --app X --host S --json`: container release records newest first with current/rollback_available (+reason); static apps list retained releases — both via the envelope, the folder-static legacy bare array is unchanged), rollback-result (`teploy rollback [--to X] --json`: {app, server, from, to, ok}; failures keep the error envelope) and redeploy-result (`teploy redeploy [--json]`: {app, server, home, release, image, ok}). Fixtures GENERATED from the real DTOs. version-handshake advertises `secret-stdin-v1` (secret set KEY --stdin; folderless secret set/get/list/rm/rotate), `releases-json-v1`, `rollback-json-v1`, `redeploy-v1` (fresh deploy of the current release's recorded image, env re-resolved) and `env-config-guard-v1` (env set/unset refuse keys a Teploy-home config declares) (regenerated mi2.json). No MI bump. || 13 | main (server-authority spec P2 support) | 2 | Additive. New artifact config-merge-envelope (`teploy config merge --base --file - --json`: merged config + conflicting keys, read-only), fixture GENERATED from the real configMergeDTO. version-handshake advertises `config-merge-v1`, `config-unprivileged-v1` (`config ... --unprivileged` refuses host-reaching changes) and `deploy-home-v1` (`deploy --app X --image T --home teploy` creates a new Teploy-home app) (regenerated mi2.json). No MI bump. | | 12 | main (server-authority spec P1 tail) | 2 | Additive. release-record gains optional `source_sha256` (folder-home deploys store the deployed teploy.yml, detected secrets masked, as meta/.source.yml 0600) with a new valid fixture. version-handshake advertises `lock-wait-v1` (global `--wait `; lock refusals name the holder) (regenerated mi2.json). `config-home-v1` now also covers `config home teploy --from-server`, `init --from-server` and `template install --home teploy`. DR bundles are schema 2 (config home + server data); they are not a wire artifact of this corpus. No MI bump. | | 11 | main (server-authority spec P1, config home) | 2 | Additive. New artifacts config-envelope (`teploy config get|home|set --json`: home folder/teploy, config commit, applied_commit of the running release, content) and config-history-envelope (`config history --json`), fixtures GENERATED from the real DTOs (teploy-home with config changed since deploy, folder-home, two-commit history). app-list-envelope gains optional `undeployed[]` (`--include-undeployed`, v3 tombstones) with a new valid fixture; release-record gains optional `config_commit` and `actor` with a new valid fixture. version-handshake advertises `config-home-v1` and `undeployed-list-v1` (regenerated mi2.json). State schema v3 (v2 + `tombstone`) marks Teploy-home apps; it is server state, not a wire artifact, and older CLIs refuse it by design. No MI bump. | | 10 | main (server-authority spec P1, events) | 2 | Additive. New artifact events-envelope: `teploy events --json` (per-app server event history, `/deployments/.teploy/apps//events.jsonl`), schema + one valid fixture GENERATED from the real `writeEventsJSON` encoder (started/succeeded deploy pair + key-names-only env.set). version-handshake advertises `events-v1` and `actor-flag-v1` (regenerated mi2.json). No MI bump. | @@ -45,10 +45,13 @@ Neutron/Nucleus dependency and a public mirror. | config-envelope | yes (rev 11) | valid x2 (teploy-home, folder-home; real configDTO) | teploy-cli | | config-history-envelope | yes (rev 11) | valid (real configHistoryDTO) | teploy-cli | | config-merge-envelope | yes (rev 13) | valid (real configMergeDTO) | teploy-cli | +| backup-target-envelope | yes (rev 15) | valid x2 (configured, unconfigured; real backupTargetDTO) | teploy-cli | +| backup-list-envelope | yes (rev 15) | valid x2 (app, legacy-schedule; real backupListDTO) | teploy-cli | +| backup-create-envelope | yes (rev 15) | valid x2 (volumes, accessory; real backupCreateDTO) | teploy-cli | +| backup-restore-envelope | yes (rev 15) | valid x2 (ok, unhealthy; real backupRestoreDTO) | teploy-cli | | releases-envelope | yes (rev 14) | valid x2 (container app, static app; real releasesDTO) | teploy-cli | | rollback-result | yes (rev 14) | valid (real rollbackResult) | teploy-cli | | redeploy-result | yes (rev 14) | valid x2 (teploy-home, folder-home; real redeployDTO) | teploy-cli | - ## Rules - Fixtures under `valid/` and `legacy/` are GENERATED from the real diff --git a/contracts/fixtures/backup-create-envelope/valid/accessory.json b/contracts/fixtures/backup-create-envelope/valid/accessory.json new file mode 100644 index 0000000..e480c95 --- /dev/null +++ b/contracts/fixtures/backup-create-envelope/valid/accessory.json @@ -0,0 +1,12 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "app": "blog", + "id": "20261020-031704-1f2e3d4c5b6a7988", + "kind": "accessory", + "accessory": "db", + "size_bytes": 2097152, + "created_at": "2026-10-20T03:17:04Z", + "key": "s3://teploy-backups/blog/accessories/db/20261020-031704-1f2e3d4c5b6a7988.sql.gz", + "pruned": 0 +} diff --git a/contracts/fixtures/backup-create-envelope/valid/volumes.json b/contracts/fixtures/backup-create-envelope/valid/volumes.json new file mode 100644 index 0000000..59c9a0b --- /dev/null +++ b/contracts/fixtures/backup-create-envelope/valid/volumes.json @@ -0,0 +1,12 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "app": "blog", + "id": "20261020-031704-1f2e3d4c5b6a7988", + "kind": "volumes", + "size_bytes": 1048576, + "created_at": "2026-10-20T03:17:04Z", + "key": "s3://teploy-backups/blog/volumes/20261020-031704-1f2e3d4c5b6a7988.tar.gz", + "teploy_key": "s3://teploy-backups/blog/teploy/20261020-031704-1f2e3d4c5b6a7988.tar.gz", + "pruned": 1 +} diff --git a/contracts/fixtures/backup-list-envelope/valid/app.json b/contracts/fixtures/backup-list-envelope/valid/app.json new file mode 100644 index 0000000..7009663 --- /dev/null +++ b/contracts/fixtures/backup-list-envelope/valid/app.json @@ -0,0 +1,53 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "app": "blog", + "target": { + "bucket": "teploy-backups", + "region": "us-east-1", + "has_credentials": true + }, + "backups": [ + { + "id": "20261020-031704-1f2e3d4c5b6a7988", + "kind": "volumes", + "size_bytes": 1048576, + "created_at": "2026-10-20T03:17:04Z", + "key": "s3://teploy-backups/blog/volumes/20261020-031704-1f2e3d4c5b6a7988.tar.gz", + "restorable": true + }, + { + "id": "20261020-031704-1f2e3d4c5b6a7988", + "kind": "teploy", + "size_bytes": 4096, + "created_at": "2026-10-20T03:17:04Z", + "key": "s3://teploy-backups/blog/teploy/20261020-031704-1f2e3d4c5b6a7988.tar.gz", + "restorable": true + }, + { + "id": "20261019-030001", + "kind": "accessory", + "accessory": "db", + "size_bytes": 2097152, + "created_at": "2026-10-19T03:00:01Z", + "key": "s3://teploy-backups/blog/accessories/db/20261019-030001.sql.gz", + "restorable": true + } + ], + "schedules": [ + { + "kind": "volumes", + "cron": "30 3 * * *", + "keep_last": 7, + "form": "run" + }, + { + "kind": "accessory", + "accessory": "db", + "cron": "15 2 * * *", + "keep_last": 5, + "max_age_days": 30, + "form": "run" + } + ] +} diff --git a/contracts/fixtures/backup-list-envelope/valid/legacy-schedule.json b/contracts/fixtures/backup-list-envelope/valid/legacy-schedule.json new file mode 100644 index 0000000..ce4d3a9 --- /dev/null +++ b/contracts/fixtures/backup-list-envelope/valid/legacy-schedule.json @@ -0,0 +1,14 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "app": "legacy", + "backups": [], + "schedules": [ + { + "kind": "volumes", + "cron": "0 3 * * *", + "keep_last": 7, + "form": "legacy" + } + ] +} diff --git a/contracts/fixtures/backup-restore-envelope/valid/ok.json b/contracts/fixtures/backup-restore-envelope/valid/ok.json new file mode 100644 index 0000000..1ada7e3 --- /dev/null +++ b/contracts/fixtures/backup-restore-envelope/valid/ok.json @@ -0,0 +1,17 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "app": "blog", + "id": "20261019-030001", + "kind": "volumes", + "ok": true, + "stopped": [ + "blog-web-1f2e3d4", + "blog-worker-1f2e3d4" + ], + "started": [ + "blog-web-1f2e3d4", + "blog-worker-1f2e3d4" + ], + "health": "healthy" +} diff --git a/contracts/fixtures/backup-restore-envelope/valid/unhealthy.json b/contracts/fixtures/backup-restore-envelope/valid/unhealthy.json new file mode 100644 index 0000000..3e69141 --- /dev/null +++ b/contracts/fixtures/backup-restore-envelope/valid/unhealthy.json @@ -0,0 +1,18 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "app": "blog", + "id": "20261019-030001", + "kind": "accessory", + "accessory": "db", + "ok": false, + "stopped": [ + "blog-web-1f2e3d4" + ], + "started": [ + "blog-web-1f2e3d4" + ], + "health": "unhealthy", + "recovery_dir": "/deployments/blog/volumes.restore-old.a1b2c3", + "error": "health check after restore failed: ... — pre-restore data kept in /deployments/blog/volumes.restore-old.a1b2c3" +} diff --git a/contracts/fixtures/backup-target-envelope/valid/configured.json b/contracts/fixtures/backup-target-envelope/valid/configured.json new file mode 100644 index 0000000..7bf229a --- /dev/null +++ b/contracts/fixtures/backup-target-envelope/valid/configured.json @@ -0,0 +1,11 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "configured": true, + "bucket": "teploy-backups", + "region": "us-east-1", + "endpoint": "http://127.0.0.1:9100", + "prefix": "prod", + "has_credentials": true, + "path": "/etc/teploy/backup-target.json" +} diff --git a/contracts/fixtures/backup-target-envelope/valid/unconfigured.json b/contracts/fixtures/backup-target-envelope/valid/unconfigured.json new file mode 100644 index 0000000..f1adadf --- /dev/null +++ b/contracts/fixtures/backup-target-envelope/valid/unconfigured.json @@ -0,0 +1,6 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "configured": false, + "has_credentials": false +} diff --git a/contracts/fixtures/version-handshake/valid/mi2.json b/contracts/fixtures/version-handshake/valid/mi2.json index af56c4d..a3ff53f 100644 --- a/contracts/fixtures/version-handshake/valid/mi2.json +++ b/contracts/fixtures/version-handshake/valid/mi2.json @@ -3,6 +3,8 @@ "actor-flag-v1", "app-list-machine", "autodeploy-redeploy", + "backup-json-v1", + "backup-target-v1", "config-home-v1", "config-merge-v1", "config-unprivileged-v1", diff --git a/contracts/schema/backup-create-envelope.schema.json b/contracts/schema/backup-create-envelope.schema.json new file mode 100644 index 0000000..86990b0 --- /dev/null +++ b/contracts/schema/backup-create-envelope.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/backup-create-envelope.schema.json", + "title": "`teploy backup create --json`: the created backup's identity (backup-json-v1, rev 14).", + "type": "object", + "required": ["machine_interface", "host", "app", "id", "kind", "size_bytes", "created_at", "key", "pruned"], + "properties": { + "machine_interface": {"type": "integer", "minimum": 1}, + "host": {"type": "string", "minLength": 1}, + "app": {"type": "string", "minLength": 1}, + "id": {"type": "string"}, + "kind": {"enum": ["volumes", "accessory"]}, + "accessory": {"type": "string"}, + "size_bytes": {"type": "integer", "minimum": 0, "description": "Size of the uploaded volumes archive or engine dump. 0 when the size could not be stated (the backup itself succeeded)."}, + "created_at": {"type": "string", "description": "RFC3339 UTC derived from the backup id's timestamp."}, + "key": {"type": "string", "pattern": "^s3://"}, + "teploy_key": {"type": "string", "pattern": "^s3://", "description": "Sibling object holding the app's server config/history (volume backups only, when present)."}, + "pruned": {"type": "integer", "minimum": 0, "description": "Backups removed by the requested retention policy after the upload."} + } +} diff --git a/contracts/schema/backup-list-envelope.schema.json b/contracts/schema/backup-list-envelope.schema.json new file mode 100644 index 0000000..9ea3813 --- /dev/null +++ b/contracts/schema/backup-list-envelope.schema.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/backup-list-envelope.schema.json", + "title": "`teploy backup list --json`: parsed backups across the volumes, accessories and teploy prefixes plus schedules discovered from teploy cron markers (backup-json-v1, rev 14).", + "type": "object", + "required": ["machine_interface", "host", "app", "backups", "schedules"], + "properties": { + "machine_interface": {"type": "integer", "minimum": 1}, + "host": {"type": "string", "minLength": 1}, + "app": {"type": "string", "minLength": 1}, + "target": { + "type": "object", + "description": "The stored server target in force (absent when the legacy flags overrode it).", + "required": ["bucket", "region", "has_credentials"], + "properties": { + "bucket": {"type": "string", "minLength": 1}, + "region": {"type": "string", "minLength": 1}, + "endpoint": {"type": "string"}, + "prefix": {"type": "string"}, + "has_credentials": {"type": "boolean"} + } + }, + "backups": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "kind", "size_bytes", "created_at", "key", "restorable"], + "properties": { + "id": {"type": "string", "description": "Backup identity: 20060102-150405, optionally suffixed -0123456789abcdef."}, + "kind": {"enum": ["volumes", "accessory", "teploy"]}, + "accessory": {"type": "string", "description": "Accessory name; present only when kind is accessory."}, + "size_bytes": {"type": "integer", "minimum": 0}, + "created_at": {"type": "string", "description": "RFC3339 UTC derived from the backup id's timestamp."}, + "key": {"type": "string", "pattern": "^s3://"}, + "restorable": {"type": "boolean", "description": "Whether the id satisfies the restore grammar (teploy rows are listed but not restorable through `backup restore`)."} + } + } + }, + "schedules": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "cron", "form"], + "properties": { + "kind": {"enum": ["volumes", "accessory"]}, + "accessory": {"type": "string"}, + "cron": {"type": "string", "description": "The five cron schedule fields."}, + "keep_last": {"type": "integer", "minimum": 0}, + "max_age_days": {"type": "integer", "minimum": 0}, + "form": {"enum": ["run", "legacy", "custom"], "description": "run: `backup run` form; legacy: pre-D3 teploy-generated (bucket/credentials embedded); custom: hand-written command under a teploy marker, left alone by rewrites."} + } + } + } + } +} diff --git a/contracts/schema/backup-restore-envelope.schema.json b/contracts/schema/backup-restore-envelope.schema.json new file mode 100644 index 0000000..62ef245 --- /dev/null +++ b/contracts/schema/backup-restore-envelope.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/backup-restore-envelope.schema.json", + "title": "`teploy backup restore --json`: the quiesced restore receipt (backup-json-v1, rev 14).", + "type": "object", + "required": ["machine_interface", "host", "app", "id", "kind", "ok", "stopped", "started", "health"], + "properties": { + "machine_interface": {"type": "integer", "minimum": 1}, + "host": {"type": "string", "minLength": 1}, + "app": {"type": "string", "minLength": 1}, + "id": {"type": "string"}, + "kind": {"enum": ["volumes", "accessory"]}, + "accessory": {"type": "string"}, + "ok": {"type": "boolean"}, + "stopped": {"type": "array", "items": {"type": "string"}, "description": "Containers stopped for the restore (the app's processes, plus a generic accessory's container for accessory restores)."}, + "started": {"type": "array", "items": {"type": "string"}}, + "health": {"enum": ["healthy", "unhealthy", "skipped"], "description": "Post-restore health check of the app's web process; skipped when nothing was stopped (undeployed app)."}, + "recovery_dir": {"type": "string", "description": "Present when the pre-restore data copy was KEPT (health failed after promotion, or a recovery-incomplete failure): the operator's way back."}, + "error": {"type": "string"} + } +} diff --git a/contracts/schema/backup-target-envelope.schema.json b/contracts/schema/backup-target-envelope.schema.json new file mode 100644 index 0000000..5da3557 --- /dev/null +++ b/contracts/schema/backup-target-envelope.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/backup-target-envelope.schema.json", + "title": "`teploy server backup-target show --json`: the server's stored backup target (backup-target-v1, rev 14). Never carries secret material — only whether any is stored.", + "type": "object", + "required": ["machine_interface", "host", "configured", "has_credentials"], + "properties": { + "machine_interface": {"type": "integer", "minimum": 1}, + "host": {"type": "string", "minLength": 1}, + "configured": {"type": "boolean"}, + "bucket": {"type": "string", "minLength": 1}, + "region": {"type": "string", "minLength": 1}, + "endpoint": {"type": "string", "pattern": "^https?://"}, + "prefix": {"type": "string", "minLength": 1}, + "has_credentials": {"type": "boolean", "description": "Whether age-encrypted credentials are stored on the server. The envelope never carries credential values."}, + "path": {"type": "string", "description": "Server path the target was read from (/etc/teploy/backup-target.json or the ~/.config/teploy fallback)."} + } +} diff --git a/internal/backup/backup.go b/internal/backup/backup.go index e48598d..253ff15 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -165,34 +165,66 @@ func unsignedDecimal(s string) bool { return true } -// S3Config holds S3 bucket and credentials info (stored on server). +// S3Config holds S3 bucket and credentials info. // // Endpoint (optional) points at an S3-compatible server instead of AWS — // e.g. a self-hosted MinIO accessory (`http://-minio:9000`) or // B2/R2/Bunny. AccessKey/SecretKey (optional) are passed inline to the aws // CLI call as env vars for that command only, never written to disk on the -// server — omit them to use the server's ambient AWS credentials as before. +// server. StoredTarget (optional) names the server's backup-target file +// (D3): when set, the encrypted credentials it carries are decrypted and +// exported inside the invocation itself and Prefix is part of the key +// layout (s3://///...). type S3Config struct { Bucket string Region string Endpoint string AccessKey string SecretKey string + // Prefix is an optional key-layout prefix under the bucket. + Prefix string + // StoredTarget is the server path of backup-target.json whose encrypted + // credentials should back this config (set when it was resolved from + // the server's stored target rather than flags). + StoredTarget string } -// AWS builds an aws-CLI invocation for this config: inline credential env -// (if set), the subcommand, then --region and --endpoint-url as configured. +// Base is the bucket-root-relative key prefix every backup key is built +// from: the bare bucket, or bucket/prefix when a prefix is configured. +func (s3 S3Config) Base() string { + if s3.Prefix == "" { + return s3.Bucket + } + return s3.Bucket + "/" + s3.Prefix +} + +// AWS builds an aws-CLI invocation for this config. Inline credential env +// (if set) or the stored-target decrypt prelude comes first, then the +// subcommand, then --region and --endpoint-url as configured. func (s3 S3Config) AWS(args string) string { var b strings.Builder - if s3.AccessKey != "" { - b.WriteString("AWS_ACCESS_KEY_ID=" + ssh.ShellQuote(s3.AccessKey) + " ") - } - if s3.SecretKey != "" { - b.WriteString("AWS_SECRET_ACCESS_KEY=" + ssh.ShellQuote(s3.SecretKey) + " ") + awsCmd := func() string { + s := "aws " + args + " --region " + ssh.ShellQuote(s3.Region) + if s3.Endpoint != "" { + s += " --endpoint-url " + ssh.ShellQuote(s3.Endpoint) + } + return s } - b.WriteString("aws " + args + " --region " + ssh.ShellQuote(s3.Region)) - if s3.Endpoint != "" { - b.WriteString(" --endpoint-url " + ssh.ShellQuote(s3.Endpoint)) + switch { + case s3.AccessKey != "": + if s3.AccessKey != "" { + b.WriteString("AWS_ACCESS_KEY_ID=" + ssh.ShellQuote(s3.AccessKey) + " ") + } + if s3.SecretKey != "" { + b.WriteString("AWS_SECRET_ACCESS_KEY=" + ssh.ShellQuote(s3.SecretKey) + " ") + } + b.WriteString(awsCmd()) + case s3.StoredTarget != "": + b.WriteString(StoredCredsPrefix(s3.StoredTarget)) + b.WriteString(awsCmd()) + b.WriteString(" )") + default: + b.WriteString(awsCmd()) } return b.String() } @@ -208,38 +240,45 @@ func NewClient(exec ssh.Executor, out io.Writer) *Client { return &Client{exec: exec, out: out} } -// The app's .env rides inside the archive as a top-level `.env` member, -// which RestoreVolumes puts back beside the volumes directory. The old -// command passed the absolute .env path as a second tar argument: tar -// strips the leading slash, so restore unpacked it to -// volumes/deployments//.env — a nested host-path artifact nobody -// ever read back, while the live .env stayed stale. -// -// The archive is built inside a private 0700 mktemp workspace: it contains -// the app's .env and volume data, and the old predictable /tmp path exposed -// it to other local users under the default umask (audit F34). -func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) error { +// BackupResult describes one backup as created (D3's backup-json-v1 rows). +type BackupResult struct { + ID string + Key string + SizeBytes int64 + // TeployKey is the sibling server-config object for volume backups, + // empty when the app has no .teploy directory (or for accessories). + TeployKey string +} + +// BackupVolumes archives the app's volumes (and .env) to S3, reporting the +// created backup's identity and size. The .env rides inside the archive as +// a top-level member which restore puts back beside the volumes directory; +// the archive is built inside a private 0700 mktemp workspace (audit F34). +func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) (BackupResult, error) { + res := BackupResult{} if err := c.ensureAWSCLI(ctx); err != nil { - return err + return res, err } timestamp, err := newBackupID(time.Now()) if err != nil { - return err + return res, err } + res.ID = timestamp volumesDir := fmt.Sprintf("%s/%s/volumes", deploymentsDir, app) appDir := fmt.Sprintf("%s/%s", deploymentsDir, app) runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-backup.XXXXXXXX") if err != nil { - return fmt.Errorf("creating backup workspace: %w", err) + return res, fmt.Errorf("creating backup workspace: %w", err) } workDir := strings.TrimSpace(runOut) cleanup := func() { c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(workDir)) } archivePath := workDir + "/backup.tar.gz" - s3Key := fmt.Sprintf("s3://%s/%s/volumes/%s.tar.gz", s3.Bucket, app, timestamp) + s3Key := fmt.Sprintf("s3://%s/%s/volumes/%s.tar.gz", s3.Base(), app, timestamp) + res.Key = s3Key // An explicit if/else, not `tar ... || tar ...`: the old fallback // re-ran the tar without .env whenever the first tar failed for ANY @@ -253,7 +292,10 @@ func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) err ) if _, err := c.exec.Run(ctx, cmd); err != nil { cleanup() - return fmt.Errorf("creating archive: %w", err) + return res, fmt.Errorf("creating archive: %w", err) + } + if size, err := c.remoteSize(ctx, archivePath); err == nil { + res.SizeBytes = size } // Upload to S3. @@ -261,7 +303,7 @@ func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) err uploadCmd := s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(archivePath), ssh.ShellQuote(s3Key))) if _, err := c.exec.Run(ctx, uploadCmd); err != nil { cleanup() - return fmt.Errorf("uploading to S3: %w", err) + return res, fmt.Errorf("uploading to S3: %w", err) } // The app's server data (Teploy config repository and event history) @@ -270,31 +312,57 @@ func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) err teployDir := confighome.Dir(app) if present, err := c.remoteExists(ctx, teployDir); err != nil { cleanup() - return fmt.Errorf("checking %s: %w", teployDir, err) + return res, fmt.Errorf("checking %s: %w", teployDir, err) } else if present { teployArchive := workDir + "/teploy.tar.gz" - teployKey := fmt.Sprintf("s3://%s/%s/teploy/%s.tar.gz", s3.Bucket, app, timestamp) + teployKey := fmt.Sprintf("s3://%s/%s/teploy/%s.tar.gz", s3.Base(), app, timestamp) + res.TeployKey = teployKey tarCmd := fmt.Sprintf("tar -czf %s --exclude=./.write.lock --exclude=./events.lock .", ssh.ShellQuote(teployArchive)) if _, err := c.exec.Run(ctx, fmt.Sprintf("cd %s && if [ -d config.git ] && command -v flock >/dev/null 2>&1; then flock .write.lock %s; else %s; fi", ssh.ShellQuote(teployDir), tarCmd, tarCmd)); err != nil { cleanup() - return fmt.Errorf("archiving %s: %w", teployDir, err) + return res, fmt.Errorf("archiving %s: %w", teployDir, err) } fmt.Fprintf(c.out, "Uploading Teploy config and history to %s...\n", teployKey) if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(teployArchive), ssh.ShellQuote(teployKey)))); err != nil { cleanup() - return fmt.Errorf("uploading Teploy config to S3: %w", err) + return res, fmt.Errorf("uploading Teploy config to S3: %w", err) } } cleanup() fmt.Fprintf(c.out, "Backup complete: %s\n", s3Key) - return nil + return res, nil +} + +// remoteSize stats a file on the server. A stat failure is not fatal to the +// backup — the size is reporting, not correctness. +func (c *Client) remoteSize(ctx context.Context, path string) (int64, error) { + out, err := c.exec.Run(ctx, "stat -c %s "+ssh.ShellQuote(path)) + if err != nil { + return 0, err + } + return strconv.ParseInt(strings.TrimSpace(out), 10, 64) } // RestoreVolumes downloads and extracts a volume backup from S3. If the // archive carries the app-level .env (see BackupVolumes), it is restored // beside the volumes directory in the app directory — the pre-restore file // is kept as .env.pre-restore so overwriting credentials is recoverable. +func (c *Client) RestoreVolumes(ctx context.Context, app, date string, s3 S3Config) error { + recoveryDir, err := c.RestoreVolumesKeep(ctx, app, date, s3, "") + if err == nil && recoveryDir != "" { + c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(recoveryDir)) + } + return err +} + +// RestoreVolumesKeep is RestoreVolumes where the caller owns the recovery +// directory: on success it is returned (empty when none was created) instead +// of being deleted, so a caller with post-restore steps of its own (D3's +// quiesced restore health check) can keep the pre-restore copy until those +// steps also pass. quiesced, when non-empty, names the containers that were +// stopped for this restore — recorded into failure messages so the operator +// knows what is down. // // Every scratch path (download, staging, staged env) lives under one private // mktemp run directory: the old fixed /tmp names were shared by concurrent @@ -303,22 +371,22 @@ func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) err // staged env (audit F33). The promotion's recovery directory is retained // until the .env commit also succeeds, so volume data and env are replaced // as one recoverable unit (audit F35). -func (c *Client) RestoreVolumes(ctx context.Context, app, date string, s3 S3Config) error { +func (c *Client) RestoreVolumesKeep(ctx context.Context, app, date string, s3 S3Config, quiesced string) (string, error) { if err := ValidateDate(date); err != nil { - return err + return "", err } if err := c.ensureAWSCLI(ctx); err != nil { - return err + return "", err } volumesDir := fmt.Sprintf("%s/%s/volumes", deploymentsDir, app) appDir := fmt.Sprintf("%s/%s", deploymentsDir, app) envPath := appDir + "/.env" - s3Key := fmt.Sprintf("s3://%s/%s/volumes/%s.tar.gz", s3.Bucket, app, date) + s3Key := fmt.Sprintf("s3://%s/%s/volumes/%s.tar.gz", s3.Base(), app, date) runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-volume-restore.XXXXXXXX") if err != nil { - return fmt.Errorf("creating restore workspace: %w", err) + return "", fmt.Errorf("creating restore workspace: %w", err) } runDir := strings.TrimSpace(runOut) cleanupRun := func() { @@ -331,7 +399,7 @@ func (c *Client) RestoreVolumes(ctx context.Context, app, date string, s3 S3Conf fmt.Fprintf(c.out, "Downloading %s...\n", s3Key) if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(s3Key), ssh.ShellQuote(archivePath)))); err != nil { cleanupRun() - return fmt.Errorf("downloading from S3: %w", err) + return "", fmt.Errorf("downloading from S3: %w", err) } // Before the staged tree is promoted into the volumes directory, pull @@ -362,17 +430,14 @@ func (c *Client) RestoreVolumes(ctx context.Context, app, date string, s3 S3Conf if !errors.Is(err, errRecoveryIncomplete) { cleanupRun() } - return err - } - // The volumes promotion succeeded but is not yet committed: keep the - // recovery directory until the .env step below also succeeds, then - // release both together. - dropRecovery := func() { - if recoveryDir != "" { - c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(recoveryDir)) + if quiesced != "" { + err = fmt.Errorf("%w (containers stopped for the restore: %s — restart them or re-run after recovering)", err, quiesced) } - cleanupRun() + return recoveryDir, err } + // The volumes promotion succeeded but is not yet committed: the + // recovery directory is returned to the caller, which releases it only + // once its own post-restore steps also succeed. // Install the backed-up .env (if the archive carried one — older // backups and volumes-only schedules don't) while keeping the previous @@ -403,13 +468,16 @@ func (c *Client) RestoreVolumes(ctx context.Context, app, date string, s3 S3Conf if out, err := c.exec.Run(ctx, envCmd); err != nil { // Env commit failed: keep the recovery dir + run dir so the mixed // state is manually recoverable, and say exactly that. - return fmt.Errorf("restoring .env: %w — volume recovery copy kept at %s, restore files kept in %s", err, recoveryDir, runDir) + return recoveryDir, fmt.Errorf("restoring .env: %w — volume recovery copy kept at %s, restore files kept in %s", err, recoveryDir, runDir) } else if strings.TrimSpace(out) != "" { fmt.Fprint(c.out, out+"\n") } - dropRecovery() + // The run directory is spent; the recovery directory stays alive for + // the caller (the RestoreVolumes wrapper drops it once the whole + // command succeeds). + cleanupRun() fmt.Fprintln(c.out, "Restore complete") - return nil + return recoveryDir, nil } // extractToStagingThenPromote extracts archivePath into a fresh, isolated @@ -550,7 +618,7 @@ func (c *Client) ListBackups(ctx context.Context, app, prefix string, s3 S3Confi return nil, err } - s3Path := fmt.Sprintf("s3://%s/%s/%s/", s3.Bucket, app, prefix) + s3Path := fmt.Sprintf("s3://%s/%s/%s/", s3.Base(), app, prefix) output, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 ls %s", ssh.ShellQuote(s3Path)))) if err != nil { return nil, fmt.Errorf("listing backups: %w", err) @@ -573,20 +641,22 @@ func (c *Client) ListBackups(ctx context.Context, app, prefix string, s3 S3Confi // inside one private 0700 mktemp workspace and removed on every exit — the // old predictable /tmp/-… paths were shared across accessories of the // same app and exposed database dumps to other local users (audit F34). -func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, env map[string]string, s3 S3Config) error { +func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, env map[string]string, s3 S3Config) (BackupResult, error) { + res := BackupResult{} if err := c.ensureAWSCLI(ctx); err != nil { - return err + return res, err } timestamp, err := newBackupID(time.Now()) if err != nil { - return err + return res, err } + res.ID = timestamp containerName := app + "-" + name runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-backup.XXXXXXXX") if err != nil { - return fmt.Errorf("creating backup workspace: %w", err) + return res, fmt.Errorf("creating backup workspace: %w", err) } workDir := strings.TrimSpace(runOut) cleanup := func() { @@ -598,34 +668,48 @@ func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, e // can never drift apart. plan := planAccessoryDump(app, name, image, env, workDir) dumpPath := plan.artifactPath - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s%s", s3.Bucket, app, name, timestamp, plan.ext) + s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s%s", s3.Base(), app, name, timestamp, plan.ext) + res.Key = s3Key if err := plan.stage(ctx, c.exec); err != nil { cleanup() - return fmt.Errorf("staging the mysql credential file for %s (no dump was run): %w", name, err) + return res, fmt.Errorf("staging the mysql credential file for %s (no dump was run): %w", name, err) } fmt.Fprintf(c.out, "Backing up %s...\n", containerName) if _, err := c.exec.Run(ctx, plan.cmd); err != nil { // Never leave partial backups around: they're disk-fillers at best, // restore-bait at worst. cleanup() - return fmt.Errorf("dumping %s: %w", name, err) + return res, fmt.Errorf("dumping %s: %w", name, err) + } + if size, err := c.remoteSize(ctx, dumpPath); err == nil { + res.SizeBytes = size } fmt.Fprintf(c.out, "Uploading to %s...\n", s3Key) if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(dumpPath), ssh.ShellQuote(s3Key)))); err != nil { cleanup() - return fmt.Errorf("uploading to S3: %w", err) + return res, fmt.Errorf("uploading to S3: %w", err) } cleanup() fmt.Fprintf(c.out, "Backup complete: %s\n", s3Key) - return nil + return res, nil } // AccessoryRestore restores a database backup from S3. func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date string, env map[string]string, s3 S3Config) error { + recoveryDir, err := c.AccessoryRestoreKeep(ctx, app, name, image, date, env, s3) + if err == nil && recoveryDir != "" { + c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(recoveryDir)) + } + return err +} + +// AccessoryRestoreKeep is AccessoryRestore where the caller owns the +// (generic-tar branch's) recovery directory — see RestoreVolumesKeep. +func (c *Client) AccessoryRestoreKeep(ctx context.Context, app, name, image, date string, env map[string]string, s3 S3Config) (string, error) { if err := c.ensureAWSCLI(ctx); err != nil { - return err + return "", err } // All restore scratch files live under one per-invocation temp dir. The @@ -636,21 +720,22 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st // kept and named in the error for inspection; on success it is removed. tmpOut, err := c.exec.Run(ctx, "mktemp -d "+ssh.ShellQuote("/tmp/teploy-restore.XXXXXX")) if err != nil { - return fmt.Errorf("creating restore temp dir: %w", err) + return "", fmt.Errorf("creating restore temp dir: %w", err) } tmpdir := strings.TrimSpace(tmpOut) - keepTmp := func(err error) error { - return fmt.Errorf("%w — restore files kept in %s", err, tmpdir) + keepTmp := func(err error) (string, error) { + return "", fmt.Errorf("%w — restore files kept in %s", err, tmpdir) } // Determine file type and restore command based on DB type. The generic // (tar) branch leaves restoreCmd empty and instead sets accDir, since it // goes through the staging helper below rather than a single shell command. var s3Key, restorePath, restoreCmd, accDir string + var genericRecovery string switch { case isDBType(image, "postgres"): - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Bucket, app, name, date) + s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Base(), app, name, date) restorePath = tmpdir + "/restore.sql.gz" db, user := postgresDBAndUser(app, env) restoreCmd = postgresRestoreCmd(app+"-"+name, user, db, restorePath, tmpdir+"/restore.sql") @@ -658,7 +743,7 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st // The old branch never assigned s3Key/restorePath, so the download // and gunzip below operated on empty arguments — every MySQL/MariaDB // restore was deterministically broken (audit F31). - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Bucket, app, name, date) + s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Base(), app, name, date) restorePath = tmpdir + "/restore.sql.gz" qContainer := ssh.ShellQuote(app + "-" + name) db := mysqlDB(app, env) @@ -684,12 +769,12 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st // The kept-on-failure scratch dir must never keep the credential. innerKeep := keepTmp - keepTmp = func(err error) error { + keepTmp = func(err error) (string, error) { c.exec.Run(context.WithoutCancel(ctx), "rm -f "+ssh.ShellQuote(tmpdir+"/mysql.env")) return innerKeep(err) } case isDBType(image, "mongo"): - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.archive.gz", s3.Bucket, app, name, date) + s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.archive.gz", s3.Base(), app, name, date) restorePath = tmpdir + "/restore.archive.gz" restoreCmd = mongoRestoreCmd(app+"-"+name, restorePath) case isDBType(image, "redis"): @@ -698,7 +783,7 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st // redis restores always failed. Stop redis first so its shutdown // save can't overwrite the snapshot we copy in, then start so it // loads dump.rdb (full reasoning on redisRestoreScript). - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.rdb.gz", s3.Bucket, app, name, date) + s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.rdb.gz", s3.Base(), app, name, date) restorePath = tmpdir + "/restore.rdb.gz" if err := c.redisAOFPreflight(ctx, app, name); err != nil { return keepTmp(err) @@ -706,7 +791,7 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st restoreCmd = redisRestoreScript(app+"-"+name, restorePath, tmpdir+"/restore.rdb", tmpdir+"/old-dump.rdb") default: // Generic: extract tar to accessory directory. - s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Bucket, app, name, date) + s3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Base(), app, name, date) restorePath = tmpdir + "/restore.tar.gz" accDir = fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, app, name) } @@ -734,18 +819,17 @@ func (c *Client) AccessoryRestore(ctx context.Context, app, name, image, date st // so the retained paths stay inspectable (TCL-43). return keepTmp(fmt.Errorf("restoring %s: %w", name, err)) } - // No dependent steps follow a generic accessory restore, so the - // retained recovery directory can be released immediately. - if recoveryDir != "" { - c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(recoveryDir)) - } + // The recovery directory is returned to the caller (the + // AccessoryRestore wrapper releases it once the whole command + // succeeds); the staged tree in tmpdir is spent either way. + genericRecovery = recoveryDir } else if _, err := c.exec.Run(ctx, restoreCmd); err != nil { return keepTmp(fmt.Errorf("restoring %s: %w", name, err)) } c.exec.Run(ctx, "rm -rf "+ssh.ShellQuote(tmpdir)) fmt.Fprintln(c.out, "Restore complete") - return nil + return genericRecovery, nil } // SetSchedule creates (or replaces) a cron job for scheduled backups. marker is diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 7bf34d1..45affec 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -28,7 +28,7 @@ func TestBackupVolumes(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) - err := client.BackupVolumes(context.Background(), "myapp", S3Config{ + _, err := client.BackupVolumes(context.Background(), "myapp", S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -86,7 +86,7 @@ func TestAccessoryBackup_Postgres(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) - err := client.AccessoryBackup(context.Background(), "myapp", "postgres", "postgres:16", nil, S3Config{ + _, err := client.AccessoryBackup(context.Background(), "myapp", "postgres", "postgres:16", nil, S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -132,7 +132,7 @@ func TestAccessoryBackup_Postgres_CustomDBName(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) env := map[string]string{"POSTGRES_DB": "realdbname", "POSTGRES_USER": "customuser"} - err := client.AccessoryBackup(context.Background(), "myapp", "postgres", "postgres:16", env, S3Config{ + _, err := client.AccessoryBackup(context.Background(), "myapp", "postgres", "postgres:16", env, S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -174,7 +174,7 @@ func TestAccessoryBackup_Postgres_DumpFailureIsNotSwallowed(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) - err := client.AccessoryBackup(context.Background(), "myapp", "postgres", "postgres:16", nil, S3Config{ + _, err := client.AccessoryBackup(context.Background(), "myapp", "postgres", "postgres:16", nil, S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -200,7 +200,7 @@ func TestAccessoryBackup_MySQL(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) - err := client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", nil, S3Config{ + _, err := client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", nil, S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -233,7 +233,7 @@ func TestAccessoryBackup_RedisWaitsForBgsave(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) - err := client.AccessoryBackup(context.Background(), "myapp", "redis", "redis:7", nil, S3Config{ + _, err := client.AccessoryBackup(context.Background(), "myapp", "redis", "redis:7", nil, S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -284,7 +284,7 @@ func TestAccessoryBackup_MySQL_UsesRootPasswordEnv(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) env := map[string]string{"MYSQL_ROOT_PASSWORD": "sekret"} - err := client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", env, S3Config{ + _, err := client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", env, S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -329,7 +329,7 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { ssh.MockCommand{Match: "rm -f", Output: ""}, ) client := NewClient(mock, &buf) - err := client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", + _, err := client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", map[string]string{"MYSQL_PASSWORD": "fall"}, S3Config{Bucket: "my-bucket", Region: "us-east-1"}) if err != nil { t.Fatalf("AccessoryBackup: %v", err) @@ -359,7 +359,7 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { ssh.MockCommand{Match: "rm -f", Output: ""}, ) client = NewClient(mock, &buf) - err = client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", nil, + _, err = client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", nil, S3Config{Bucket: "my-bucket", Region: "us-east-1"}) if err != nil { t.Fatalf("AccessoryBackup: %v", err) @@ -386,7 +386,7 @@ func TestAccessoryBackup_MySQL_QuotesHostilePassword(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) env := map[string]string{"MYSQL_ROOT_PASSWORD": "p@'ss word; id"} - err := client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", env, S3Config{ + _, err := client.AccessoryBackup(context.Background(), "myapp", "mysql", "mysql:8", env, S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -550,7 +550,7 @@ func TestAccessoryBackup_Generic(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) - err := client.AccessoryBackup(context.Background(), "myapp", "meilisearch", "meilisearch:latest", nil, S3Config{ + _, err := client.AccessoryBackup(context.Background(), "myapp", "meilisearch", "meilisearch:latest", nil, S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -638,7 +638,7 @@ func TestEnsureAWSCLI_Missing(t *testing.T) { var buf bytes.Buffer client := NewClient(mock, &buf) - err := client.BackupVolumes(context.Background(), "myapp", S3Config{ + _, err := client.BackupVolumes(context.Background(), "myapp", S3Config{ Bucket: "my-bucket", Region: "us-east-1", }) @@ -979,7 +979,7 @@ func TestBackupVolumesIncludesTeployServerData(t *testing.T) { ssh.MockCommand{Match: "if [ -e '/deployments/.teploy/apps/myapp' ]", Output: "present\n"}, ssh.MockCommand{Match: "cd '/deployments/.teploy/apps/myapp'", Output: ""}, ) - if err := NewClient(mock, &bytes.Buffer{}).BackupVolumes(context.Background(), "myapp", S3Config{Bucket: "my-bucket", Region: "us-east-1"}); err != nil { + if _, err := NewClient(mock, &bytes.Buffer{}).BackupVolumes(context.Background(), "myapp", S3Config{Bucket: "my-bucket", Region: "us-east-1"}); err != nil { t.Fatalf("BackupVolumes: %v", err) } var archived, uploaded bool diff --git a/internal/backup/bundle.go b/internal/backup/bundle.go index ce36839..0f5685e 100644 --- a/internal/backup/bundle.go +++ b/internal/backup/bundle.go @@ -249,7 +249,7 @@ type BundleStore interface { type S3BundleStore struct{ S3 S3Config } func (s S3BundleStore) key(app, id, member string) string { - return fmt.Sprintf("s3://%s/%s/dr/%s/%s", s.S3.Bucket, app, id, member) + return fmt.Sprintf("s3://%s/%s/dr/%s/%s", s.S3.Base(), app, id, member) } func (s S3BundleStore) Upload(ctx context.Context, exec ssh.Executor, app, id, member, serverPath string) error { @@ -280,7 +280,7 @@ func (s S3BundleStore) FetchManifest(ctx context.Context, exec ssh.Executor, app } func (s S3BundleStore) ListIDs(ctx context.Context, exec ssh.Executor, app string) ([]string, error) { - prefix := fmt.Sprintf("s3://%s/%s/dr/", s.S3.Bucket, app) + prefix := fmt.Sprintf("s3://%s/%s/dr/", s.S3.Base(), app) out, err := exec.Run(ctx, s.S3.AWS(fmt.Sprintf("s3 ls %s", ssh.ShellQuote(prefix)))) if err != nil { return nil, fmt.Errorf("listing bundles at %s: %w", prefix, err) diff --git a/internal/backup/cron.go b/internal/backup/cron.go new file mode 100644 index 0000000..320d594 --- /dev/null +++ b/internal/backup/cron.go @@ -0,0 +1,108 @@ +package backup + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "time" +) + +// Cron-marker discovery (D3): teploy-managed backup cron lines end in a +// bracketed marker comment; their schedule, form and baked-in retention +// are parsed for `backup list --json`, `backup-target set`'s rewrite and +// `teploy doctor`. + +const ( + CronFormRun = "run" // ` backup run --app X [--accessory Y]` + CronFormLegacy = "legacy" // pre-D3 generated forms (bucket/creds embedded) + CronFormCustom = "custom" // hand-written command under a teploy marker +) + +// CronMarker is one parsed teploy backup cron line. +type CronMarker struct { + Kind string // volumes | accessory + App string + Accessory string + Cron string // the five schedule fields + Form string + Tag string // the bare marker (teploy-backup:app or teploy-accessory-backup:app:name) + KeepLast int // legacy lines only + MaxAgeDays int + Command string +} + +var cronMarkerRE = regexp.MustCompile(`\[(teploy-backup:[A-Za-z0-9._-]+|teploy-accessory-backup:[A-Za-z0-9._-]+:[A-Za-z0-9._-]+)\]\s*$`) + +var legacyKeepLastRE = regexp.MustCompile(`head -n -([0-9]+)`) + +// FindCronMarker parses one crontab line; nil when the line carries no +// teploy backup marker. +func FindCronMarker(line string) *CronMarker { + m := cronMarkerRE.FindStringSubmatch(line) + if m == nil { + return nil + } + tag := m[1] + fields := strings.Fields(line) + if len(fields) < 6 { + return nil + } + mk := &CronMarker{Tag: tag, Cron: strings.Join(fields[:5], " "), Command: strings.Join(fields[5:], " ")} + if rest, ok := strings.CutPrefix(tag, "teploy-accessory-backup:"); ok { + app, acc, found := strings.Cut(rest, ":") + if !found { + return nil + } + mk.Kind, mk.App, mk.Accessory = KindAccessory, app, acc + } else { + mk.Kind, mk.App = KindVolumes, strings.TrimPrefix(tag, "teploy-backup:") + } + switch { + case strings.Contains(mk.Command, " backup run --app "): + mk.Form = CronFormRun + case isLegacyGeneratedBackupCmd(mk.Command): + mk.Form = CronFormLegacy + if kb := legacyKeepLastRE.FindStringSubmatch(mk.Command); kb != nil { + mk.KeepLast = atoiSafe(kb[1]) + } + default: + mk.Form = CronFormCustom + } + return mk +} + +// isLegacyGeneratedBackupCmd recognizes the command shapes this CLI +// generated before D3: the volumes shell chain (tar + aws), the accessory +// invocation, or anything carrying a --bucket flag. A hand-written script +// under a teploy marker (the nucleus backup pattern) matches none of +// these and is reported as custom instead of rewritten. +func isLegacyGeneratedBackupCmd(cmd string) bool { + return strings.Contains(cmd, "aws s3 cp") || + strings.Contains(cmd, " accessory backup ") || + strings.Contains(cmd, " --bucket ") +} + +// IsRecoveryIncomplete reports whether err (or anything it wraps) is the +// restore failure class whose recovery artifacts are intentionally kept +// on the server — callers use it to decide whether restarting the app +// after a failed restore would serve data of unknown integrity. +func IsRecoveryIncomplete(err error) bool { + return errors.Is(err, errRecoveryIncomplete) +} + +// ParseIDTimestamp renders the timestamp part of a backup id as RFC3339 +// UTC. +func ParseIDTimestamp(stamp string) (string, error) { + t, err := time.Parse("20060102-150405", stamp) + if err != nil { + return "", fmt.Errorf("invalid backup timestamp %q", stamp) + } + return t.UTC().Format(time.RFC3339), nil +} + +// EnsureAWSCLIPublic is the exported form of the aws-CLI presence check. +func (c *Client) EnsureAWSCLIPublic(ctx context.Context) error { + return c.ensureAWSCLI(ctx) +} diff --git a/internal/backup/list.go b/internal/backup/list.go new file mode 100644 index 0000000..661cbab --- /dev/null +++ b/internal/backup/list.go @@ -0,0 +1,149 @@ +package backup + +import ( + "context" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +// Listing (D3): `backup list --json` reports parsed rows across the +// volumes, accessories and teploy prefixes, not raw `aws s3 ls` lines. + +// BackupKind names the prefix family a listed backup belongs to. +const ( + KindVolumes = "volumes" + KindAccessory = "accessory" + KindTeployData = "teploy" +) + +// ListedBackup is one parsed object under the app's S3 layout. +type ListedBackup struct { + ID string `json:"id"` + Kind string `json:"kind"` + Accessory string `json:"accessory,omitempty"` + SizeBytes int64 `json:"size_bytes"` + CreatedAt string `json:"created_at"` + Key string `json:"key"` + Restorable bool `json:"restorable"` +} + +// listedIDRE matches the object names the CLI writes: the backup-id +// grammar (timestamp or timestamp+hex) with each branch's extension. +var listedIDRE = regexp.MustCompile(`^([0-9]{8}-[0-9]{6}(?:-[0-9a-f]{16})?)\.(tar\.gz|sql\.gz|archive\.gz|rdb\.gz)$`) + +// ListBackupsRecursive lists every object under s3://// in one +// recursive call and classifies each by its key layout. Objects whose name +// does not match the CLI's naming (strays, partial uploads) are skipped. +func (c *Client) ListBackupsRecursive(ctx context.Context, app string, s3 S3Config) ([]ListedBackup, error) { + if err := c.ensureAWSCLI(ctx); err != nil { + return nil, err + } + root := fmt.Sprintf("s3://%s/%s/", s3.Base(), app) + out, err := c.exec.Run(ctx, s3.AWS("s3 ls "+ssh.ShellQuote(root)+" --recursive")) + if err != nil { + return nil, fmt.Errorf("listing backups: %w", err) + } + var rows []ListedBackup + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + // Layout: DATE TIME SIZE KEY (SIZE is "0" or a plain integer; the + // PRE pseudo-rows only appear in non-recursive listings). + size, err := strconv.ParseInt(fields[2], 10, 64) + if err != nil { + continue + } + key := fields[3] + row, ok := classifyListedKey(app, key) + if !ok { + continue + } + row.SizeBytes = size + // Recursive listing keys are relative to the BUCKET root (they + // already carry the app prefix), so rebuild the full URI from the + // base, not from the listing prefix. + row.Key = "s3://" + s3.Base() + "/" + key + rows = append(rows, row) + } + sort.SliceStable(rows, func(i, j int) bool { + if rows[i].CreatedAt != rows[j].CreatedAt { + return rows[i].CreatedAt > rows[j].CreatedAt + } + return rows[i].Key < rows[j].Key + }) + return rows, nil +} + +// classifyListedKey maps one recursive-ls key (relative to the bucket +// root, so "/volumes/.tar.gz") to a ListedBackup row. +func classifyListedKey(app, key string) (ListedBackup, bool) { + var row ListedBackup + segments := strings.Split(key, "/") + if len(segments) < 3 || segments[0] != app { + return row, false + } + m := listedIDRE.FindStringSubmatch(segments[len(segments)-1]) + if m == nil { + return row, false + } + id := m[1] + row.ID = id + if ts, err := time.Parse("20060102-150405", id[:15]); err == nil { + row.CreatedAt = ts.UTC().Format(time.RFC3339) + } + row.Restorable = ValidateDate(id) == nil + switch segments[1] { + case "volumes": + if len(segments) != 3 || m[2] != "tar.gz" { + return row, false + } + row.Kind = KindVolumes + case "teploy": + if len(segments) != 3 || m[2] != "tar.gz" { + return row, false + } + row.Kind = KindTeployData + // Listed for visibility (the server-config sibling of a volume + // backup), but `backup restore` restores volumes, not this row. + row.Restorable = false + case "accessories": + if len(segments) != 4 { + return row, false + } + row.Kind = KindAccessory + row.Accessory = segments[2] + default: + return row, false + } + return row, true +} + +// EngineOf classifies an accessory image for restore-time decisions: the +// generic (tar) branch is the only one whose restore replaces files on +// disk, so it is the only one where the accessory container itself must be +// stopped (the engine branches restore through a running engine; redis +// stops and restarts itself inside its restore script). +func EngineOf(image string) string { + switch { + case isDBType(image, "postgres"): + return "postgres" + case isDBType(image, "mysql"): + return "mysql" + case isDBType(image, "mariadb"): + return "mariadb" + case isDBType(image, "mongo"): + return "mongo" + case isDBType(image, "redis"): + return "redis" + default: + return "generic" + } +} diff --git a/internal/backup/list_test.go b/internal/backup/list_test.go new file mode 100644 index 0000000..f372df1 --- /dev/null +++ b/internal/backup/list_test.go @@ -0,0 +1,34 @@ +package backup + +import ( + "testing" +) + +// The recursive listing parser maps keys to rows and skips strays. +func TestClassifyListedKey(t *testing.T) { + row, ok := classifyListedKey("myapp", "myapp/volumes/20261020-031704-1f2e3d4c5b6a7988.tar.gz") + if !ok || row.Kind != KindVolumes || row.ID != "20261020-031704-1f2e3d4c5b6a7988" || !row.Restorable { + t.Fatalf("volumes row: %+v ok=%v", row, ok) + } + if row.CreatedAt != "2026-10-20T03:17:04Z" { + t.Fatalf("created_at = %q", row.CreatedAt) + } + if row, ok := classifyListedKey("myapp", "myapp/accessories/db/20261019-030001.sql.gz"); !ok || row.Kind != KindAccessory || row.Accessory != "db" { + t.Fatalf("accessory row: %+v ok=%v", row, ok) + } + if row, ok := classifyListedKey("myapp", "myapp/teploy/20261019-030001.tar.gz"); !ok || row.Kind != KindTeployData || row.Restorable { + t.Fatalf("teploy row: %+v ok=%v", row, ok) + } + for _, stray := range []string{ + "myapp/volumes/notabackup.tar.gz", + "myapp/volumes/20261020-031704.tar.gz.tmp", + "otherapp/volumes/20261020-031704.tar.gz", + "myapp/unknown/20261020-031704.tar.gz", + "myapp/accessories/20261020-031704.tar.gz", + "myapp/volumes/nested/20261020-031704.tar.gz", + } { + if row, ok := classifyListedKey("myapp", stray); ok { + t.Errorf("stray classified: %s -> %+v", stray, row) + } + } +} diff --git a/internal/backup/retention.go b/internal/backup/retention.go index b9ea254..d6254fa 100644 --- a/internal/backup/retention.go +++ b/internal/backup/retention.go @@ -120,7 +120,7 @@ func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Conf var deleted []string for _, name := range toDelete { - key := fmt.Sprintf("s3://%s/%s/%s/%s", s3.Bucket, app, prefix, name) + key := fmt.Sprintf("s3://%s/%s/%s/%s", s3.Base(), app, prefix, name) if _, err := c.exec.Run(ctx, s3.AWS("s3 rm "+ssh.ShellQuote(key))); err != nil { return deleted, fmt.Errorf("deleting %s: %w", name, err) } diff --git a/internal/backup/schedule.go b/internal/backup/schedule.go new file mode 100644 index 0000000..1777208 --- /dev/null +++ b/internal/backup/schedule.go @@ -0,0 +1,123 @@ +package backup + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/useteploy/teploy/internal/confighome" + "github.com/useteploy/teploy/internal/ssh" +) + +// The per-app backup schedule store (D3). Cron lines call only +// `teploy backup run --app X [--accessory Y]`, so the schedule's retention +// half lives beside the app's server data and is read at run time — +// changing retention never rewrites a crontab, and a crontab holds no +// bucket or credentials. + +// KindPolicy is the schedule+retention pair for one backup kind. +type KindPolicy struct { + Cron string `json:"cron,omitempty"` + KeepLast int `json:"keep_last,omitempty"` + MaxAgeDays int `json:"max_age_days,omitempty"` +} + +// AppPolicy is the whole per-app schedule store. +type AppPolicy struct { + Volumes *KindPolicy `json:"volumes,omitempty"` + Accessories map[string]*KindPolicy `json:"accessories,omitempty"` +} + +// PolicyPath is the server path of the app's schedule store. +func PolicyPath(app string) string { + return confighome.Dir(app) + "/backup-schedule.json" +} + +// ReadPolicy loads the app's schedule store; an absent file is an empty +// policy, not an error. +func ReadPolicy(ctx context.Context, exec ssh.Executor, app string) (AppPolicy, error) { + var p AppPolicy + res := ssh.RunDetailed(ctx, exec, "cat "+ssh.ShellQuote(PolicyPath(app))+" 2>/dev/null || true") + if res.Err != nil { + return p, fmt.Errorf("reading the backup schedule store: %w", res.Err) + } + raw := strings.TrimSpace(string(res.Stdout)) + if raw == "" { + return p, nil + } + if err := json.Unmarshal([]byte(raw), &p); err != nil { + return p, fmt.Errorf("parsing %s: %w", PolicyPath(app), err) + } + return p, nil +} + +// WritePolicy persists the app's schedule store atomically at 0600 under +// the app's .teploy directory (created if this is the first write). +func WritePolicy(ctx context.Context, exec ssh.Executor, app string, p AppPolicy) error { + payload, err := json.Marshal(p) + if err != nil { + return err + } + res := ssh.RunInputDetailed(ctx, exec, writePolicyScript(ssh.ShellQuote(PolicyPath(app))), strings.NewReader(string(payload))) + if res.Err != nil { + return fmt.Errorf("writing the backup schedule store: %w", res.Err) + } + if res.ExitCode != 0 { + return fmt.Errorf("writing the backup schedule store: %s", res.ExitErrorText()) + } + return nil +} + +func writePolicyScript(qPath string) string { + return fmt.Sprintf(`d=$(dirname %s) +mkdir -p "$d" || exit 1 +umask 077 +t=$(mktemp "$d/.backup-schedule.XXXXXXXX") || exit 1 +cat > "$t" || { rm -f -- "$t"; exit 1; } +chmod 600 "$t" || { rm -f -- "$t"; exit 1; } +mv -f -- "$t" %s || { rm -f -- "$t"; exit 1; }`, qPath, qPath) +} + +// RemoveSchedule deletes the cron entry carrying marker (the counterpart +// of SetSchedule), reading the crontab with the same checked discipline. +func (c *Client) RemoveSchedule(ctx context.Context, marker string) error { + if strings.ContainsAny(marker, "\r\n\x00") { + return fmt.Errorf("cron marker must be a single line") + } + tag := "[" + marker + "]" + cmd := fmt.Sprintf( + `raw=$(crontab -l 2>&1); rc=$?; `+ + `if [ "$rc" -ne 0 ]; then case "$raw" in *"no crontab"*) raw="";; *) `+ + `echo "reading crontab failed: $raw" >&2; exit 1;; esac; fi; `+ + `(printf '%%s\n' "$raw" | grep -vF %s; true) | crontab -`, + ssh.ShellQuote(tag), + ) + if _, err := c.exec.Run(ctx, cmd); err != nil { + return fmt.Errorf("removing cron schedule: %w", err) + } + return nil +} + +// ReadCrontab returns the user's crontab lines (nil when none). The read +// is exit-status-checked (the TCL-46 discipline): only the canonical "no +// crontab for " failure means empty; every other failure is an error. +func ReadCrontab(ctx context.Context, exec ssh.Executor) ([]string, error) { + res := ssh.RunDetailed(ctx, exec, "crontab -l") + if res.Err != nil { + return nil, fmt.Errorf("reading crontab: %w", res.Err) + } + if res.ExitCode != 0 { + if strings.Contains(string(res.Stderr), "no crontab") { + return nil, nil + } + return nil, fmt.Errorf("reading crontab: %s", res.ExitErrorText()) + } + var lines []string + for _, line := range strings.Split(strings.TrimSpace(string(res.Stdout)), "\n") { + if line = strings.TrimRight(line, "\r"); strings.TrimSpace(line) != "" { + lines = append(lines, line) + } + } + return lines, nil +} diff --git a/internal/backup/target.go b/internal/backup/target.go new file mode 100644 index 0000000..1d29904 --- /dev/null +++ b/internal/backup/target.go @@ -0,0 +1,291 @@ +package backup + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + + "github.com/useteploy/teploy/internal/ssh" +) + +// One backup target per server (D3): bucket, region, endpoint and an +// optional key prefix, stored ON the server and read at run time so no +// crontab line or laptop environment has to carry them. +// +// The file lives outside /deployments (which Caddy mounts read-only into +// untrusted containers): /etc/teploy/backup-target.json, mode 0600. A +// non-root deploy user that cannot write /etc/teploy falls back to +// ~/.config/teploy/backup-target.json and the command says so. +// +// Credentials, when given, are age-encrypted with the server's own age +// identity and stored INSIDE the JSON as the "credentials" field. The +// plaintext never touches the client's disk, argv, or the crontab; at use +// time S3Config.AWS decrypts and exports them inside the command's shell. +// The identity is resolved by try-order (/etc/teploy/age-key, then +// /deployments/.age-key) so it follows the same move the secret store's +// identity is making; a fresh identity is only created when NEITHER path +// holds one, and then at /deployments/.age-key — the pre-migration +// canonical location — so a server never grows two identities. + +const ( + // TargetFileEtc is the preferred backup-target location. + TargetFileEtc = "/etc/teploy/backup-target.json" + // TargetFileHome is the non-root fallback location. + TargetFileHome = ".config/teploy/backup-target.json" + + ageIdentityEtc = "/etc/teploy/age-key" + ageIdentityFarm = "/deployments/.age-key" +) + +// Target is the non-secret half of the stored backup target plus a flag +// saying whether encrypted credentials travel with it. +type Target struct { + Bucket string `json:"bucket"` + Region string `json:"region"` + Endpoint string `json:"endpoint,omitempty"` + Prefix string `json:"prefix,omitempty"` + // HasCredentials reports whether the stored file carries an encrypted + // credentials blob. The blob itself is never decoded into this struct. + HasCredentials bool `json:"-"` + // Path is the server path the target was read from (diagnostics only). + Path string `json:"-"` +} + +// targetFile is the on-disk shape. Credentials holds age armor (empty when +// the target relies on the server's ambient AWS credential chain). +type targetFile struct { + V string `json:"v"` + Bucket string `json:"bucket"` + Region string `json:"region"` + Endpoint string `json:"endpoint"` + Prefix string `json:"prefix"` + Credentials string `json:"credentials,omitempty"` +} + +// credentialValueRE is the charset a stored S3 access key or secret may +// use. The plaintext is evaluated by the shell at use time +// (S3Config.AWS), so anything that could end a word or quote a string is +// refused here rather than escaped. +var credentialValueRE = regexp.MustCompile(`^[A-Za-z0-9/+_.=:@-]{1,255}$`) + +// ValidateCredential refuses values the run-time eval path cannot treat as +// a single safe shell word. +func ValidateCredential(kind, value string) error { + if !credentialValueRE.MatchString(value) { + return fmt.Errorf("%s must be 1-255 characters of letters, digits and / + _ . = : @ - (no spaces, quotes or shell metacharacters)", kind) + } + return nil +} + +var endpointRE = regexp.MustCompile(`^https?://[A-Za-z0-9.-]+(:[0-9]{1,5})?(/[A-Za-z0-9._~/=-]*)?$`) + +// ValidateEndpoint accepts an http(s) URL only: the value is interpolated +// into aws invocations server-side. +func ValidateEndpoint(endpoint string) error { + if !endpointRE.MatchString(endpoint) || strings.ContainsAny(endpoint, " \t\r\n'\"\\") { + return fmt.Errorf("endpoint must be an http:// or https:// URL without spaces or quotes") + } + return nil +} + +var prefixSegmentRE = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) + +// ValidatePrefix accepts slash-joined safe segments ("/" is not a leading +// or trailing possibility). +func ValidatePrefix(prefix string) error { + if prefix == "" { + return nil + } + if len(prefix) > 200 || strings.HasPrefix(prefix, "/") || strings.HasSuffix(prefix, "/") || strings.Contains(prefix, "//") { + return fmt.Errorf("prefix must be slash-joined segments without leading/trailing slashes (max 200 chars)") + } + for _, seg := range strings.Split(prefix, "/") { + if !prefixSegmentRE.MatchString(seg) { + return fmt.Errorf("prefix segment %q must be alphanumeric with dots, dashes or underscores", seg) + } + } + return nil +} + +// ReadTarget loads the server's stored backup target. A server with no +// target is (Target{}, nil) — the normal "not configured yet" case, which +// callers turn into the command-appropriate error. +func ReadTarget(ctx context.Context, exec ssh.Executor) (Target, error) { + res := ssh.RunDetailed(ctx, exec, readTargetScript) + if res.Err != nil { + return Target{}, fmt.Errorf("reading the backup target: %w", res.Err) + } + if res.ExitCode != 0 { + return Target{}, fmt.Errorf("reading the backup target: %s", res.ExitErrorText()) + } + out := strings.TrimSpace(string(res.Stdout)) + if out == "absent" { + return Target{}, nil + } + var t Target + line, rest, _ := strings.Cut(out, "\n") + if !strings.HasPrefix(line, "path ") { + return Target{}, fmt.Errorf("reading the backup target: unrecognized output framing") + } + t.Path = strings.TrimSpace(strings.TrimPrefix(line, "path ")) + var f targetFile + if err := json.Unmarshal([]byte(rest), &f); err != nil { + return Target{}, fmt.Errorf("parsing %s: %w", t.Path, err) + } + if f.V != "1" { + return Target{}, fmt.Errorf("%s: unsupported backup target version %q", t.Path, f.V) + } + t.Bucket, t.Region, t.Endpoint, t.Prefix = f.Bucket, f.Region, f.Endpoint, f.Prefix + t.HasCredentials = f.Credentials != "" + return t, nil +} + +// readTargetScript prints "absent" or "path

\n". One +// framed read (exit-status-checked, absence distinguished from failure). +const readTargetScript = `for p in /etc/teploy/backup-target.json "$HOME"/.config/teploy/backup-target.json; do +if [ -r "$p" ]; then printf 'path %s\n' "$p"; cat "$p"; exit 0; fi +done +echo absent` + +// WriteTarget stores the target (and, when creds is non-empty, the +// age-encrypted credentials blob) on the server and returns the path +// written plus whether the non-root fallback location was used. +func WriteTarget(ctx context.Context, exec ssh.Executor, t Target, credsArmor string) (path string, fallback bool, err error) { + f := targetFile{V: "1", Bucket: t.Bucket, Region: t.Region, Endpoint: t.Endpoint, Prefix: t.Prefix, Credentials: credsArmor} + payload, err := json.Marshal(f) + if err != nil { + return "", false, err + } + res := ssh.RunInputDetailed(ctx, exec, writeTargetScript, strings.NewReader(string(payload))) + if res.Err != nil { + return "", false, fmt.Errorf("writing the backup target: %w", res.Err) + } + if res.ExitCode != 0 { + return "", false, fmt.Errorf("writing the backup target: %s", res.ExitErrorText()) + } + out := strings.Fields(strings.TrimSpace(string(res.Stdout))) + // "written " on stdout; "fallback" on stderr selects the report. + fallback = strings.Contains(string(res.Stderr), "fallback") + if len(out) != 2 || out[0] != "written" { + return "", fallback, fmt.Errorf("writing the backup target: unrecognized output framing") + } + return out[1], fallback, nil +} + +// writeTargetScript reads the JSON payload from stdin and publishes it +// atomically at 0600. Root (or a user with a writable /etc/teploy) writes +// the preferred location; anyone else falls back to ~/.config/teploy and +// the script says "fallback" on stderr so the command can tell the user. +const writeTargetScript = `d= +if [ "$(id -u)" = 0 ]; then + mkdir -p /etc/teploy && chmod 700 /etc/teploy || exit 1 + d=/etc/teploy/backup-target.json +elif [ -w /etc/teploy ] 2>/dev/null; then + d=/etc/teploy/backup-target.json +else + dir=${XDG_CONFIG_HOME:-$HOME/.config}/teploy + mkdir -p "$dir" && chmod 700 "$dir" || exit 1 + d=$dir/backup-target.json + echo fallback >&2 +fi +umask 077 +t=$(mktemp "$(dirname "$d")/.backup-target.XXXXXXXX") || exit 1 +cat > "$t" || { rm -f -- "$t"; exit 1; } +chmod 600 "$t" || { rm -f -- "$t"; exit 1; } +mv -f -- "$t" "$d" || { rm -f -- "$t"; exit 1; } +printf 'written %s\n' "$d"` + +// UnsetTarget removes the stored target from every location it may occupy. +// It reports the paths removed (empty when nothing was configured). +func UnsetTarget(ctx context.Context, exec ssh.Executor) ([]string, error) { + res := ssh.RunDetailed(ctx, exec, unsetTargetScript) + if res.Err != nil { + return nil, fmt.Errorf("removing the backup target: %w", res.Err) + } + if res.ExitCode != 0 { + return nil, fmt.Errorf("removing the backup target: %s", res.ExitErrorText()) + } + var removed []string + for _, line := range strings.Split(strings.TrimSpace(string(res.Stdout)), "\n") { + if line = strings.TrimSpace(line); strings.HasPrefix(line, "removed ") { + removed = append(removed, strings.TrimSpace(strings.TrimPrefix(line, "removed "))) + } + } + return removed, nil +} + +const unsetTargetScript = `for p in /etc/teploy/backup-target.json "$HOME"/.config/teploy/backup-target.json; do +if [ -e "$p" ]; then rm -f -- "$p" || exit 1; echo "removed $p"; fi +done` + +// EncryptCredentials encrypts the two-line credentials payload (the +// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY export lines S3Config.AWS +// evaluates) to the server's age identity and returns the armor. The +// plaintext travels only over the SSH channel as stdin. +func EncryptCredentials(ctx context.Context, exec ssh.Executor, accessKey, secretKey string) (string, error) { + if err := ValidateCredential("access key", accessKey); err != nil { + return "", err + } + if err := ValidateCredential("secret key", secretKey); err != nil { + return "", err + } + idPath, err := ensureAgeIdentity(ctx, exec) + if err != nil { + return "", err + } + recipient, err := exec.Run(ctx, "age-keygen -y "+ssh.ShellQuote(idPath)) + if err != nil { + return "", fmt.Errorf("reading the age identity's recipient: %w", err) + } + recipient = strings.TrimSpace(recipient) + if recipient == "" || strings.Contains(recipient, "\n") { + return "", fmt.Errorf("age-keygen returned an unusable recipient") + } + // export is required: the eval runs in the invocation's own shell and + // the assignments must reach the exec'd aws child (bare assignments + // would stay unexported shell variables). + plaintext := "export AWS_ACCESS_KEY_ID=" + accessKey + "\nexport AWS_SECRET_ACCESS_KEY=" + secretKey + "\n" + res := ssh.RunInputDetailed(ctx, exec, "age -r "+ssh.ShellQuote(recipient)+" -a", strings.NewReader(plaintext)) + if res.Err != nil { + return "", fmt.Errorf("encrypting the credentials: %w", res.Err) + } + if res.ExitCode != 0 { + return "", fmt.Errorf("encrypting the credentials: %s", res.ExitErrorText()) + } + return strings.TrimSpace(string(res.Stdout)), nil +} + +// ensureAgeIdentity resolves the server's age identity with the W2 move +// in mind (/etc/teploy/age-key first, /deployments/.age-key as the +// pre-migration location) and creates one at the legacy path only when +// neither exists — so a server never ends up with two identities. +func ensureAgeIdentity(ctx context.Context, exec ssh.Executor) (string, error) { + for _, p := range []string{ageIdentityEtc, ageIdentityFarm} { + out, err := exec.Run(ctx, fmt.Sprintf("if [ -r %s ]; then echo yes; else echo no; fi", ssh.ShellQuote(p))) + if err != nil { + return "", fmt.Errorf("checking %s: %w", p, err) + } + if strings.TrimSpace(out) == "yes" { + return p, nil + } + } + if _, err := exec.Run(ctx, fmt.Sprintf("umask 077; age-keygen -o %s 2>/dev/null && chmod 600 %s", ssh.ShellQuote(ageIdentityFarm), ssh.ShellQuote(ageIdentityFarm))); err != nil { + return "", fmt.Errorf("generating an age identity at %s: %w", ageIdentityFarm, err) + } + return ageIdentityFarm, nil +} + +// StoredCredsPrefix is the shell prelude S3Config.AWS uses when the config +// carries a stored target: it decrypts the credentials blob inside the +// command's own shell and exports it, failing closed when decryption +// yields nothing (so a broken target never silently falls back to the +// ambient AWS chain). The identity try-order matches ensureAgeIdentity. +func StoredCredsPrefix(jsonPath string) string { + id := `$(if [ -r /etc/teploy/age-key ]; then printf '%s' /etc/teploy/age-key; else printf '%s' /deployments/.age-key; fi)` + return fmt.Sprintf( + `( c=$(python3 -c 'import json,sys; sys.stdout.write(json.load(open(sys.argv[1])).get("credentials") or "")' %s | age -d -i "%s") && [ -n "$c" ] && eval "$c" && exec `, + ssh.ShellQuote(jsonPath), id, + ) +} diff --git a/internal/backup/verify.go b/internal/backup/verify.go index cd7bad5..41aad78 100644 --- a/internal/backup/verify.go +++ b/internal/backup/verify.go @@ -69,7 +69,7 @@ func (c *Client) InspectAccessory(ctx context.Context, app, name string) (image // by listing the S3 prefix. Returns the bare timestamp (the restore // arg) and the object's size in bytes. func (c *Client) LatestBackupDate(ctx context.Context, app, name string, s3 S3Config) (date string, size int64, err error) { - prefix := fmt.Sprintf("s3://%s/%s/accessories/%s/", s3.Bucket, app, name) + prefix := fmt.Sprintf("s3://%s/%s/accessories/%s/", s3.Base(), app, name) out, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 ls %s", ssh.ShellQuote(prefix)))) if err != nil { return "", 0, fmt.Errorf("listing backups at %s: %w", prefix, err) @@ -254,7 +254,7 @@ func (c *Client) waitReady(ctx context.Context, scratch, probe string, attempts func (c *Client) verifyPostgres(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { db, user := postgresDBAndUser(app, env) - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Bucket, app, name, res.Date) + s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Base(), app, name, res.Date) res.S3Key = s3Key dump := tmpBase + ".sql.gz" if err := c.download(ctx, s3Key, dump, s3); err != nil { @@ -301,7 +301,7 @@ func (c *Client) verifyPostgres(ctx context.Context, res *VerifyResult, app, nam func (c *Client) verifyMySQL(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { db := mysqlDB(app, env) - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Bucket, app, name, res.Date) + s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Base(), app, name, res.Date) res.S3Key = s3Key dump := tmpBase + ".sql.gz" if err := c.download(ctx, s3Key, dump, s3); err != nil { @@ -349,7 +349,7 @@ func (c *Client) verifyMySQL(ctx context.Context, res *VerifyResult, app, name, } func (c *Client) verifyRedis(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.rdb.gz", s3.Bucket, app, name, res.Date) + s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.rdb.gz", s3.Base(), app, name, res.Date) res.S3Key = s3Key dump := tmpBase + ".rdb.gz" if err := c.download(ctx, s3Key, dump, s3); err != nil { @@ -379,7 +379,7 @@ func (c *Client) verifyRedis(ctx context.Context, res *VerifyResult, app, name, } func (c *Client) verifyMongo(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.archive.gz", s3.Bucket, app, name, res.Date) + s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.archive.gz", s3.Base(), app, name, res.Date) res.S3Key = s3Key dump := tmpBase + ".archive.gz" if err := c.download(ctx, s3Key, dump, s3); err != nil { @@ -425,7 +425,7 @@ func (c *Client) verifyMongo(ctx context.Context, res *VerifyResult, app, name, // tables, and stay running. The "Restored N table(s)" boot line is the // verification metric. func (c *Client) verifyNucleus(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Bucket, app, name, res.Date) + s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Base(), app, name, res.Date) res.S3Key = s3Key dump := tmpBase + ".tar.gz" if err := c.download(ctx, s3Key, dump, s3); err != nil { @@ -471,7 +471,7 @@ func (c *Client) verifyNucleus(ctx context.Context, res *VerifyResult, app, name // verifyVolume checks a generic volume archive: integrity (a truncated or // corrupt tar fails extraction) and non-emptiness. func (c *Client) verifyVolume(ctx context.Context, res *VerifyResult, app, name string, s3 S3Config, tmpBase string) error { - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Bucket, app, name, res.Date) + s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Base(), app, name, res.Date) res.S3Key = s3Key dump := tmpBase + ".tar.gz" if err := c.download(ctx, s3Key, dump, s3); err != nil { diff --git a/internal/cli/accessory.go b/internal/cli/accessory.go index d5cd59b..c99bc7d 100644 --- a/internal/cli/accessory.go +++ b/internal/cli/accessory.go @@ -14,6 +14,7 @@ import ( "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/config" "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/events" "github.com/useteploy/teploy/internal/ssh" ) @@ -366,12 +367,15 @@ const serverTeployPath = "/usr/local/bin/teploy" func newAccessoryBackupCmd(flags *Flags) *cobra.Command { var ( - bucket string - region string - endpoint string - schedule string - appName string - local bool + bucket string + region string + endpoint string + prefix string + schedule string + appName string + local bool + keepLast int + maxAgeDays int ) cmd := &cobra.Command{ @@ -379,24 +383,30 @@ func newAccessoryBackupCmd(flags *Flags) *cobra.Command { Short: "Back up an accessory (database-aware dump)", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runAccessoryBackup(flags, appName, args[0], bucket, region, endpoint, schedule, local) + return runAccessoryBackup(flags, appName, args[0], bucket, region, endpoint, prefix, schedule, local, keepLast, maxAgeDays) }, } - cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") - cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") + cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name (default: the server's stored backup target)") + cmd.Flags().StringVar(®ion, "region", "", "AWS region (default: the stored target's region, else us-east-1)") cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") + cmd.Flags().StringVar(&prefix, "prefix", "", "key prefix under the bucket (default: the stored target's prefix)") cmd.Flags().StringVar(&schedule, "schedule", "", "cron schedule for automated backups") cmd.Flags().StringVar(&appName, "app", "", "app name — read accessory image/env from server state instead of teploy.yml") cmd.Flags().BoolVar(&local, "local", false, "run against the local docker host (the mode scheduled cron jobs use; requires --app)") + cmd.Flags().IntVar(&keepLast, "keep-last", 0, "with --schedule: keep only the N most recent backups (stored, applied at run time)") + cmd.Flags().IntVar(&maxAgeDays, "max-age-days", 0, "with --schedule: delete backups older than N days (stored, applied at run time)") return cmd } -func runAccessoryBackup(flags *Flags, appName, name, bucket, region, endpoint, schedule string, local bool) error { +func runAccessoryBackup(flags *Flags, appName, name, bucket, region, endpoint, prefix, schedule string, local bool, keepLast, maxAgeDays int) error { if local && appName == "" { return fmt.Errorf("--local requires --app (no teploy.yml exists on the server)") } + if err := backup.ValidateBucket(name); err != nil { + return fmt.Errorf("invalid accessory name %q: %w", name, err) + } var app string var accImage string @@ -417,20 +427,14 @@ func runAccessoryBackup(flags *Flags, appName, name, bucket, region, endpoint, s app = appName } - if bucket == "" { - return fmt.Errorf("--bucket is required") - } - if err := backup.ValidateBucket(bucket); err != nil { - return err - } - if err := backup.ValidateRegion(region); err != nil { - return err - } if schedule != "" { if err := backup.ValidateSchedule(schedule); err != nil { return err } } + if keepLast < 0 || maxAgeDays < 0 { + return fmt.Errorf("--keep-last and --max-age-days must be >= 0") + } ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() @@ -453,8 +457,24 @@ func runAccessoryBackup(flags *Flags, appName, name, bucket, region, endpoint, s } defer executor.Close() + // Target resolution (D3): legacy flags win, else the server's stored + // target. Scheduling additionally bootstraps a stored target from the + // legacy flags when none exists, so the cron line never carries one. + tf := backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint, prefix: prefix} + var target backup.Target + if schedule != "" { + var err error + target, err = resolveScheduleTarget(ctx, executor, &tf) + if err != nil { + return err + } + } + s3Cfg, err := resolveBackupS3(ctx, executor, &tf) + if err != nil { + return err + } + client := backup.NewClient(executor, os.Stdout) - s3Cfg := s3Config(bucket, region, endpoint) // BOTH the manual and the --local (scheduled) path authenticate with the // accessory's REAL credentials. teploy.yml's accessory env can carry @@ -464,7 +484,7 @@ func runAccessoryBackup(flags *Flags, appName, name, bucket, region, endpoint, s // unresolved references (audit F68). if image, env, inspErr := client.InspectAccessory(ctx, app, name); inspErr == nil { accImage, accEnv = image, env - } else { + } else if !local { if err := requireResolvedAccessoryEnv(name, accEnv); err != nil { return fmt.Errorf("accessory %s is not inspectable (%v) and its teploy.yml env cannot be used directly: %w", name, inspErr, err) } @@ -472,38 +492,60 @@ func runAccessoryBackup(flags *Flags, appName, name, bucket, region, endpoint, s } if schedule != "" { - // The scheduled job runs ON the server: --local --app makes it - // self-contained (no teploy.yml server-side), reading accessory - // config from the running container. - // serverTeployPath, not a bare "teploy": cron runs with PATH=/usr/bin:/bin - // and the binary is installed to /usr/local/bin (below), so a bare name - // resolves to nothing. That failed silently every night — the job has no - // output redirect, so "command not found" went to a mailbox nobody reads. - // Found live 2026-08-31: a nightly accessory backup scheduled on 2026-07-12 - // had produced its last artifact on 2026-07-13, and the two that existed - // were made by hand at setup. The schedule had never once run. - backupCmd := fmt.Sprintf("%s accessory backup %s --local --app %s --bucket %s --region %s", - serverTeployPath, name, app, bucket, region) - if endpoint != "" { - // Cron has no TEPLOY_S3_* env — embed endpoint + creds in the - // crontab line (root-only readable, same trust class as - // ~/.aws/credentials). - backupCmd = fmt.Sprintf("TEPLOY_S3_ACCESS_KEY=%s TEPLOY_S3_SECRET_KEY=%s %s --endpoint %s", - ssh.ShellQuote(s3Cfg.AccessKey), ssh.ShellQuote(s3Cfg.SecretKey), backupCmd, ssh.ShellQuote(endpoint)) + // The scheduled job runs ON the server as + // `teploy backup run --app A --accessory N`: self-contained, no + // teploy.yml, no bucket, no credentials in the crontab (D3). + binPath, err := ensureScheduledBackupBinary(ctx, executor) + if err != nil { + return err } - // The cron job needs the teploy binary on the server; ship it via - // the existing checksum-verified pipeline (same as heal/autodeploy). - if _, err := deployTeployBinaryToServer(ctx, executor, serverTeployPath); err != nil { - return fmt.Errorf("installing teploy binary for the scheduled job: %w", err) + policy, err := backup.ReadPolicy(ctx, executor, app) + if err != nil { + return err + } + if policy.Accessories == nil { + policy.Accessories = map[string]*backup.KindPolicy{} + } + policy.Accessories[name] = &backup.KindPolicy{Cron: schedule, KeepLast: keepLast, MaxAgeDays: maxAgeDays} + if err := backup.WritePolicy(ctx, executor, app, policy); err != nil { + return err } - if err := client.SetSchedule(ctx, schedule, backupCmd, "teploy-accessory-backup:"+app+":"+name); err != nil { + if err := client.SetSchedule(ctx, schedule, buildScheduledBackupRunCmd(binPath, app, name), "teploy-accessory-backup:"+app+":"+name); err != nil { return err } fmt.Printf("Scheduled backup: %s\n", schedule) + fmt.Printf(" Target: s3://%s (read at run time from the server's stored target)\n", backupTargetLabel(target)) return nil } - return client.AccessoryBackup(ctx, app, name, accImage, accEnv, s3Cfg) + res, err := client.AccessoryBackup(ctx, app, name, accImage, accEnv, s3Cfg) + detail := map[string]any{"id": res.ID, "kind": backup.KindAccessory, "accessory": name} + recordAppEvent(ctx, executor, app, events.KindBackupRun, err, detail) + if err != nil { + return err + } + fmt.Printf("Backup %s created (accessory, %d bytes)\n", res.ID, res.SizeBytes) + return nil +} + +// resolveScheduleTarget is the schedule commands' target gate: a stored +// target must exist afterwards (bootstrapping one from legacy flags when +// none is stored), and legacy flags must agree with an existing one. +func resolveScheduleTarget(ctx context.Context, executor ssh.Executor, tf *backupTargetFlags) (backup.Target, error) { + target, err := backup.ReadTarget(ctx, executor) + if err != nil { + return backup.Target{}, err + } + if !TargetConfigured(target) { + if tf.bucket == "" { + return backup.Target{}, fmt.Errorf("no backup target is configured for this server — set one with `teploy server backup-target set --host S --bucket B` (or pass the legacy --bucket flags to store one now)") + } + return storeTargetFromLegacyFlags(ctx, executor, tf) + } + if tf.bucket != "" && tf.bucket != target.Bucket { + return backup.Target{}, fmt.Errorf("a server backup target already exists (s3://%s) — change it with `teploy server backup-target set`, not schedule flags", target.Bucket) + } + return target, nil } func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { @@ -511,6 +553,7 @@ func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { bucket string region string endpoint string + prefix string ) cmd := &cobra.Command{ @@ -518,18 +561,19 @@ func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { Short: "Restore an accessory from backup", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - return runAccessoryRestore(flags, args[0], args[1], bucket, region, endpoint) + return runAccessoryRestore(flags, args[0], args[1], bucket, region, endpoint, prefix) }, } - cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") - cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") + cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name (default: the server's stored backup target)") + cmd.Flags().StringVar(®ion, "region", "", "AWS region (default: the stored target's region, else us-east-1)") cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") + cmd.Flags().StringVar(&prefix, "prefix", "", "key prefix under the bucket (default: the stored target's prefix)") return cmd } -func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint string) error { +func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint, prefix string) error { appCfg, err := loadAppCfgForAccessory() if err != nil { return err @@ -540,15 +584,6 @@ func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint stri return fmt.Errorf("accessory %q not found in teploy.yml", name) } - if bucket == "" { - return fmt.Errorf("--bucket is required") - } - if err := backup.ValidateBucket(bucket); err != nil { - return err - } - if err := backup.ValidateRegion(region); err != nil { - return err - } if err := backup.ValidateDate(date); err != nil { return err } @@ -562,8 +597,11 @@ func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint stri } defer executor.Close() + s3Cfg, err := resolveBackupS3(ctx, executor, &backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint, prefix: prefix}) + if err != nil { + return err + } client := backup.NewClient(executor, os.Stdout) - s3Cfg := s3Config(bucket, region, endpoint) // Same credential rule as manual backup: prefer the RUNNING accessory's // env (the config's `auto`/`secret:` references are never what the // database actually runs with) — audit F68. @@ -631,15 +669,6 @@ func newAccessoryVerifyBackupCmd(flags *Flags) *cobra.Command { } func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, endpoint string, local bool) error { - if bucket == "" { - return fmt.Errorf("--bucket is required") - } - if err := backup.ValidateBucket(bucket); err != nil { - return err - } - if err := backup.ValidateRegion(region); err != nil { - return err - } if date != "" { if err := backup.ValidateDate(date); err != nil { return err @@ -672,13 +701,18 @@ func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, } defer executor.Close() + s3Cfg, err := resolveBackupS3(ctx, executor, &backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint}) + if err != nil { + return err + } + out := io.Writer(os.Stdout) if flags.JSON { // Keep stdout clean for the JSON result; progress goes to stderr. out = os.Stderr } client := backup.NewClient(executor, out) - res, err := client.VerifyBackup(ctx, appCfg.App, name, date, s3Config(bucket, region, endpoint)) + res, err := client.VerifyBackup(ctx, appCfg.App, name, date, s3Cfg) if err != nil { return err } diff --git a/internal/cli/accessory_schedule_test.go b/internal/cli/accessory_schedule_test.go index c41ac02..6b6737a 100644 --- a/internal/cli/accessory_schedule_test.go +++ b/internal/cli/accessory_schedule_test.go @@ -11,47 +11,58 @@ import ( // cron runs with PATH=/usr/bin:/bin. The CLI installs itself to // /usr/local/bin/teploy for exactly these scheduled jobs — and then, for a // year, wrote a cron line that called a bare `teploy`, which resolves to -// nothing under that PATH. Because the generated line carries no output +// nothing under that PATH. Because the generated line carried no output // redirect either, "command not found" went to a mailbox nobody reads. // // Found live on 2026-08-31: a nightly Nucleus accessory backup scheduled on -// 2026-07-12 had produced its last artifact on 2026-07-13. The two objects in -// the bucket were both made by hand at setup. The schedule had never once run, -// and the failure was invisible from every surface — `crontab -l` showed the -// job, cron was active, and the command was correct in every respect except -// the one that mattered. +// 2026-07-12 had produced its last artifact on 2026-07-13. The two objects +// in the bucket were both made by hand at setup. The schedule had never +// run, and the failure was invisible from every surface — `crontab -l` +// showed the job, cron was active, and the command was correct in every +// respect except the one that mattered. // // A source test because the alternative is a live cron and a day of waiting. +// Since D3 the line is `backup run` (see backup_run.go); the same absolute +// path rule applies, plus: no bucket, no credentials. func TestScheduledAccessoryBackupUsesAnAbsolutePath(t *testing.T) { - src, err := os.ReadFile("accessory.go") + src, err := os.ReadFile("backup_run.go") if err != nil { t.Fatal(err) } body := string(src) - i := strings.Index(body, "backupCmd := fmt.Sprintf(") + i := strings.Index(body, "func buildScheduledBackupRunCmd") if i < 0 { t.Fatal("the scheduled backup command is no longer built here; move this test with it") } - // The whole statement, not the first line: the format string and its - // arguments are on separate lines, and the path is an argument. line := body[i:] - if j := strings.Index(line, ")\n"); j > 0 { + if j := strings.Index(line, "\n}\n"); j > 0 { line = line[:j] } - if strings.Contains(line, `"teploy accessory backup`) { + if strings.Contains(line, `"teploy `) { t.Error("the generated cron line invokes a bare `teploy`; cron's PATH is /usr/bin:/bin " + "and the binary is installed to /usr/local/bin, so the job silently does nothing") } - if !strings.Contains(line, "serverTeployPath") { - t.Error("the generated cron line must use serverTeployPath") - } if serverTeployPath != "/usr/local/bin/teploy" { t.Errorf("serverTeployPath = %q; it must match where deployTeployBinaryToServer installs it", serverTeployPath) } // The install and the invocation must name the SAME path — that they // disagreed is the whole defect. - if !strings.Contains(body, "deployTeployBinaryToServer(ctx, executor, serverTeployPath)") { + if !strings.Contains(body, "deployTeployBinaryToServer(ctx, executor, serverTeployPath, \"backup run\")") { t.Error("the binary is installed to a path the generated command does not reference") } } + +// The generated accessory schedule line must not embed the bucket or the +// credentials — both come from the server's stored target at run time. +func TestScheduledAccessoryLineCarriesNoTarget(t *testing.T) { + line := buildScheduledBackupRunCmd(serverTeployPath, "myapp", "db") + for _, banned := range []string{"--bucket", "TEPLOY_S3", "AWS_SECRET", "AWS_ACCESS"} { + if strings.Contains(line, banned) { + t.Errorf("generated cron line embeds %s: %s", banned, line) + } + } + if !strings.Contains(line, "backup run --app 'myapp' --accessory 'db'") { + t.Errorf("generated cron line is not the run form: %s", line) + } +} diff --git a/internal/cli/backup.go b/internal/cli/backup.go index 916229e..3631957 100644 --- a/internal/cli/backup.go +++ b/internal/cli/backup.go @@ -3,7 +3,9 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" + "io" "os" "os/signal" "strings" @@ -11,10 +13,23 @@ import ( "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/events" "github.com/useteploy/teploy/internal/notify" + secretstore "github.com/useteploy/teploy/internal/secret" "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" ) +// The backup family (D3): one target per server, read at run time. +// +// Every command resolves its S3 target the same way — explicit legacy +// flags (--bucket and friends) win, else the server's stored target +// (`teploy server backup-target set`). Folderless operation (--app X +// --host S) works everywhere: volumes and .teploy data live on the +// server, and accessory facts come from the running container. + func newBackupCmd(flags *Flags) *cobra.Command { cmd := &cobra.Command{ Use: "backup", @@ -26,49 +41,98 @@ func newBackupCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newBackupRestoreCmd(flags)) cmd.AddCommand(newBackupPruneCmd(flags)) cmd.AddCommand(newBackupScheduleCmd(flags)) + cmd.AddCommand(newBackupUnscheduleCmd(flags)) + cmd.AddCommand(newBackupRunCmd(flags)) return cmd } -// addRetentionFlags registers the shared --keep-last / --max-age-days flags. -func addRetentionFlags(cmd *cobra.Command, keepLast, maxAgeDays *int) { - cmd.Flags().IntVar(keepLast, "keep-last", 0, "Keep only the N most recent backups (0 = keep all)") - cmd.Flags().IntVar(maxAgeDays, "max-age-days", 0, "Delete backups older than N days (0 = no age limit; pair with --keep-last as a floor)") +// backupTargetFlags carries the shared S3-target flags. An empty Bucket +// means "not given" (resolve from the server's stored target). +type backupTargetFlags struct { + bucket string + region string + endpoint string + prefix string } -func newBackupCreateCmd(flags *Flags) *cobra.Command { - var ( - bucket string - region string - endpoint string - keepLast int - maxAgeDays int - ) +// addBackupTargetFlags registers the legacy target flags. They override +// the stored server target when given (existing users' scripts keep +// working unchanged). +func addBackupTargetFlags(cmd *cobra.Command, f *backupTargetFlags) { + cmd.Flags().StringVar(&f.bucket, "bucket", "", "S3 bucket name (default: the server's stored backup target)") + cmd.Flags().StringVar(&f.region, "region", "", "AWS region (default: the stored target's region, else us-east-1)") + cmd.Flags().StringVar(&f.endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") + cmd.Flags().StringVar(&f.prefix, "prefix", "", "key prefix under the bucket (default: the stored target's prefix)") +} - cmd := &cobra.Command{ - Use: "create", - Short: "Create a volume backup", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return runBackupCreate(flags, bucket, region, endpoint, keepLast, maxAgeDays) - }, +// resolveBackupS3 builds the S3Config for a backup command. Legacy flags +// win (unchanged legacy behavior including env-credential gating); with no +// flags the server's stored target is used, stored credentials and all. +func resolveBackupS3(ctx context.Context, exec ssh.Executor, f *backupTargetFlags) (backup.S3Config, error) { + if f.bucket != "" { + if err := backup.ValidateBucket(f.bucket); err != nil { + return backup.S3Config{}, err + } + region := f.region + if region == "" { + region = "us-east-1" + } + if err := backup.ValidateRegion(region); err != nil { + return backup.S3Config{}, err + } + if f.endpoint != "" { + if err := backup.ValidateEndpoint(f.endpoint); err != nil { + return backup.S3Config{}, err + } + } + if err := backup.ValidatePrefix(f.prefix); err != nil { + return backup.S3Config{}, err + } + cfg := s3Config(f.bucket, region, f.endpoint) + cfg.Prefix = f.prefix + return cfg, nil + } + target, err := backup.ReadTarget(ctx, exec) + if err != nil { + return backup.S3Config{}, err + } + if !TargetConfigured(target) { + return backup.S3Config{}, fmt.Errorf("no backup target is configured for this server — set one with `teploy server backup-target set --host S --bucket B` (or pass the legacy --bucket flag)") } + return TargetS3(target), nil +} - cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") - cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") - cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") - addRetentionFlags(cmd, &keepLast, &maxAgeDays) +// TargetConfigured reports whether a stored target exists. +func TargetConfigured(t backup.Target) bool { + return t.Bucket != "" +} - return cmd +// TargetS3 maps a stored target to an S3Config. Stored credentials apply +// whether or not an endpoint is set (D3); the ambient AWS chain is used +// only when none are stored. +func TargetS3(t backup.Target) backup.S3Config { + region := t.Region + if region == "" { + region = "us-east-1" + } + return backup.S3Config{ + Bucket: t.Bucket, + Region: region, + Endpoint: t.Endpoint, + Prefix: t.Prefix, + StoredTarget: t.Path, + } } -// s3Config assembles the backup target. --endpoint switches the server-side -// aws CLI to an S3-compatible endpoint (a MinIO accessory, B2, R2, Bunny). -// Credentials come from the LOCAL environment — TEPLOY_S3_ACCESS_KEY / -// TEPLOY_S3_SECRET_KEY, falling back to AWS_ACCESS_KEY_ID / -// AWS_SECRET_ACCESS_KEY — and ride inline on the remote aws command for -// that invocation only; nothing is persisted server-side. Unset = the -// server's ambient AWS credentials, exactly as before. +// s3Config assembles the legacy flag-driven target. --endpoint switches the +// server-side aws CLI to an S3-compatible endpoint (a MinIO accessory, B2, +// R2, Bunny). Credentials come from the LOCAL environment — +// TEPLOY_S3_ACCESS_KEY / TEPLOY_S3_SECRET_KEY, falling back to +// AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — and ride inline on the +// remote aws command for that invocation only; nothing is persisted +// server-side. Unset = the server's ambient AWS credentials, exactly as +// before. func s3Config(bucket, region, endpoint string) backup.S3Config { access := os.Getenv("TEPLOY_S3_ACCESS_KEY") if access == "" { @@ -86,47 +150,130 @@ func s3Config(bucket, region, endpoint string) backup.S3Config { return backup.S3Config{Bucket: bucket, Region: region, Endpoint: endpoint, AccessKey: access, SecretKey: secret} } -func runBackupCreate(flags *Flags, bucket, region, endpoint string, keepLast, maxAgeDays int) error { - appCfg, err := config.LoadApp(".") - if err != nil { - return err - } +// addRetentionFlags registers the shared --keep-last / --max-age-days flags. +func addRetentionFlags(cmd *cobra.Command, keepLast, maxAgeDays *int) { + cmd.Flags().IntVar(keepLast, "keep-last", 0, "Keep only the N most recent backups (0 = keep all)") + cmd.Flags().IntVar(maxAgeDays, "max-age-days", 0, "Delete backups older than N days (0 = no age limit; pair with --keep-last as a floor)") +} - if bucket == "" { - return fmt.Errorf("--bucket is required") +// backupProgressOut keeps stdout clean for the JSON envelope. +func backupProgressOut(flags *Flags) io.Writer { + if flags.JSON { + return os.Stderr } - if err := backup.ValidateBucket(bucket); err != nil { - return err + return os.Stdout +} + +// backupCreatedAt renders a backup id's timestamp as RFC3339 (best effort). +func backupCreatedAt(id string) string { + if len(id) >= 15 { + if ts, err := backup.ParseIDTimestamp(id[:15]); err == nil { + return ts + } } - if err := backup.ValidateRegion(region); err != nil { - return err + return "" +} + +// backupCreateDTO is the `backup create --json` envelope (backup-json-v1). +type backupCreateDTO struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + App string `json:"app"` + ID string `json:"id"` + Kind string `json:"kind"` + Accessory string `json:"accessory,omitempty"` + SizeBytes int64 `json:"size_bytes"` + CreatedAt string `json:"created_at"` + Key string `json:"key"` + TeployKey string `json:"teploy_key,omitempty"` + Pruned int `json:"pruned"` +} + +func newBackupCreateCmd(flags *Flags) *cobra.Command { + var ( + appName string + accessory string + tf backupTargetFlags + keepLast int + maxAgeDays int + ) + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a volume or accessory backup", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runBackupCreate(flags, appName, accessory, &tf, keepLast, maxAgeDays) + }, } + cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + cmd.Flags().StringVar(&accessory, "accessory", "", "back up this accessory instead of the app's volumes") + addBackupTargetFlags(cmd, &tf) + addRetentionFlags(cmd, &keepLast, &maxAgeDays) + + return cmd +} + +func runBackupCreate(flags *Flags, appName, accessory string, tf *backupTargetFlags, keepLast, maxAgeDays int) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() - executor, err := connectForApp(ctx, flags, appCfg) + appCfg, executor, err := resolveApp(ctx, flags, appName) if err != nil { return err } defer executor.Close() - client := backup.NewClient(executor, os.Stdout) - cfg := s3Config(bucket, region, endpoint) - err = client.BackupVolumes(ctx, appCfg.App, cfg) + cfg, err := resolveBackupS3(ctx, executor, tf) + if err != nil { + return err + } + + client := backup.NewClient(executor, backupProgressOut(flags)) + + var res backup.BackupResult + var kind string + if accessory != "" { + if err := backup.ValidateBucket(accessory); err != nil { + return fmt.Errorf("invalid accessory name %q: %w", accessory, err) + } + image, env, accErr := client.InspectAccessory(ctx, appCfg.App, accessory) + if accErr != nil { + return fmt.Errorf("inspecting accessory %s: %w", accessory, accErr) + } + res, err = client.AccessoryBackup(ctx, appCfg.App, accessory, image, env, cfg) + kind = backup.KindAccessory + } else { + res, err = client.BackupVolumes(ctx, appCfg.App, cfg) + kind = backup.KindVolumes + } // Enforce retention only after a successful backup — never prune when the // fresh backup failed (that could leave zero good copies). - var pruned []string + var pruned int var pruneErr error policy := backup.RetentionPolicy{KeepLast: keepLast, MaxAgeDays: maxAgeDays} if err == nil && !policy.IsZero() { - pruned, pruneErr = client.PruneBackups(ctx, appCfg.App, "volumes", cfg, policy) + prefix := "volumes" + if accessory != "" { + prefix = "accessories/" + accessory + } + var deleted []string + deleted, pruneErr = client.PruneBackups(ctx, appCfg.App, prefix, cfg, policy) + pruned = len(deleted) + } + + detail := map[string]any{"id": res.ID, "kind": kind} + if accessory != "" { + detail["accessory"] = accessory } + recordAppEvent(ctx, executor, appCfg.App, events.KindBackupRun, err, detail) - // Fire notification (fire-and-forget). The backup itself succeeding is what - // Success reflects; a prune failure is surfaced in the message + exit code - // but does not flip the backup to "failed" (the data is safely stored). + // Fire notification (fire-and-forget). The backup itself succeeding is + // what Success reflects; a prune failure is surfaced in the message + + // exit code but does not flip the backup to "failed" (the data is + // safely stored). if n := buildNotifier(appCfg); n != nil { msg := fmt.Sprintf("Backup created for %s", appCfg.App) switch { @@ -134,8 +281,8 @@ func runBackupCreate(flags *Flags, bucket, region, endpoint string, keepLast, ma msg = fmt.Sprintf("Backup failed for %s: %s", appCfg.App, err) case pruneErr != nil: msg = fmt.Sprintf("Backup created for %s, but prune failed: %s", appCfg.App, pruneErr) - case len(pruned) > 0: - msg = fmt.Sprintf("Backup created for %s (pruned %d old backup(s))", appCfg.App, len(pruned)) + case pruned > 0: + msg = fmt.Sprintf("Backup created for %s (pruned %d old backup(s))", appCfg.App, pruned) } n.Send(ctx, notify.Payload{ App: appCfg.App, @@ -149,168 +296,575 @@ func runBackupCreate(flags *Flags, bucket, region, endpoint string, keepLast, ma if err != nil { return err } - return pruneErr + if pruneErr != nil { + return pruneErr + } + + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(backupCreateDTO{ + MachineInterface: MachineInterface, + Host: executor.Host(), + App: appCfg.App, + ID: res.ID, + Kind: kind, + Accessory: accessory, + SizeBytes: res.SizeBytes, + CreatedAt: backupCreatedAt(res.ID), + Key: res.Key, + TeployKey: res.TeployKey, + Pruned: pruned, + }) + } + fmt.Printf("Backup %s created (%s, %d bytes)\n", res.ID, kind, res.SizeBytes) + if res.TeployKey != "" { + fmt.Printf(" Server data: %s\n", res.TeployKey) + } + if pruned > 0 { + fmt.Printf(" Pruned %d old backup(s)\n", pruned) + } + return nil +} + +// backupListDTO is the `backup list --json` envelope (backup-json-v1). +type backupListDTO struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + App string `json:"app"` + Target *backupTargetSummary `json:"target,omitempty"` + Backups []backup.ListedBackup `json:"backups"` + Schedules []backupScheduleDTO `json:"schedules"` +} + +type backupTargetSummary struct { + Bucket string `json:"bucket"` + Region string `json:"region"` + Endpoint string `json:"endpoint,omitempty"` + Prefix string `json:"prefix,omitempty"` + HasCredentials bool `json:"has_credentials"` +} + +// backupScheduleDTO is one discovered schedule (from teploy cron markers, +// retention joined from the per-app schedule store). +type backupScheduleDTO struct { + Kind string `json:"kind"` + Accessory string `json:"accessory,omitempty"` + Cron string `json:"cron"` + KeepLast int `json:"keep_last,omitempty"` + MaxAgeDays int `json:"max_age_days,omitempty"` + Form string `json:"form"` } func newBackupListCmd(flags *Flags) *cobra.Command { var ( - bucket string - region string - endpoint string + appName string + tf backupTargetFlags ) cmd := &cobra.Command{ Use: "list", - Short: "List available backups", + Short: "List available backups and schedules", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runBackupList(flags, bucket, region, endpoint) + return runBackupList(flags, appName, &tf) }, } - cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") - cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") - cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") + cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + addBackupTargetFlags(cmd, &tf) return cmd } -func runBackupList(flags *Flags, bucket, region, endpoint string) error { - appCfg, err := config.LoadApp(".") +func runBackupList(flags *Flags, appName string, tf *backupTargetFlags) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + appCfg, executor, err := resolveApp(ctx, flags, appName) if err != nil { return err } + defer executor.Close() - if bucket == "" { - return fmt.Errorf("--bucket is required") + cfg, err := resolveBackupS3(ctx, executor, tf) + if err != nil { + return err } - if err := backup.ValidateBucket(bucket); err != nil { + + client := backup.NewClient(executor, backupProgressOut(flags)) + rows, err := client.ListBackupsRecursive(ctx, appCfg.App, cfg) + if err != nil { return err } - if err := backup.ValidateRegion(region); err != nil { + if rows == nil { + rows = []backup.ListedBackup{} + } + schedules, err := discoverBackupSchedules(ctx, executor, appCfg.App) + if err != nil { return err } + if schedules == nil { + schedules = []backupScheduleDTO{} + } + + if flags.JSON { + target, _ := backup.ReadTarget(ctx, executor) + var summary *backupTargetSummary + if TargetConfigured(target) { + summary = &backupTargetSummary{ + Bucket: target.Bucket, + Region: target.Region, + Endpoint: target.Endpoint, + Prefix: target.Prefix, + HasCredentials: target.HasCredentials, + } + } + return json.NewEncoder(os.Stdout).Encode(backupListDTO{ + MachineInterface: MachineInterface, + Host: executor.Host(), + App: appCfg.App, + Target: summary, + Backups: rows, + Schedules: schedules, + }) + } + + if len(rows) == 0 && len(schedules) == 0 { + fmt.Println("No backups or schedules found") + return nil + } + for _, s := range schedules { + label := s.Kind + if s.Accessory != "" { + label += "/" + s.Accessory + } + retention := "" + if s.KeepLast > 0 { + retention = fmt.Sprintf(", keep last %d", s.KeepLast) + } + if s.MaxAgeDays > 0 { + retention += fmt.Sprintf(", max age %dd", s.MaxAgeDays) + } + fmt.Printf("schedule %-20s %s (%s%s)\n", label, s.Cron, s.Form, retention) + } + for _, b := range rows { + label := b.Kind + if b.Accessory != "" { + label += "/" + b.Accessory + } + fmt.Printf("%s %-18s %10d bytes %s\n", b.CreatedAt, label, b.SizeBytes, b.Key) + } + return nil +} + +// discoverBackupSchedules reads the crontab's teploy backup markers and +// joins each with the retention stored in the app's schedule store. +func discoverBackupSchedules(ctx context.Context, exec ssh.Executor, app string) ([]backupScheduleDTO, error) { + lines, err := backup.ReadCrontab(ctx, exec) + if err != nil { + return nil, err + } + policy, err := backup.ReadPolicy(ctx, exec, app) + if err != nil { + return nil, err + } + var out []backupScheduleDTO + for _, line := range lines { + m := backup.FindCronMarker(line) + if m == nil || m.App != app { + continue + } + s := backupScheduleDTO{Kind: m.Kind, Accessory: m.Accessory, Cron: m.Cron, Form: m.Form} + var kp *backup.KindPolicy + if m.Accessory == "" { + kp = policy.Volumes + } else if policy.Accessories != nil { + kp = policy.Accessories[m.Accessory] + } + if kp != nil { + s.KeepLast, s.MaxAgeDays = kp.KeepLast, kp.MaxAgeDays + } + out = append(out, s) + } + return out, nil +} + +// backupRestoreDTO is the `backup restore --json` envelope (backup-json-v1). +type backupRestoreDTO struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + App string `json:"app"` + ID string `json:"id"` + Kind string `json:"kind"` + Accessory string `json:"accessory,omitempty"` + OK bool `json:"ok"` + Stopped []string `json:"stopped"` + Started []string `json:"started"` + Health string `json:"health"` + RecoveryDir string `json:"recovery_dir,omitempty"` + Error string `json:"error,omitempty"` +} + +func newBackupRestoreCmd(flags *Flags) *cobra.Command { + var ( + appName string + accessory string + tf backupTargetFlags + yes bool + ) + cmd := &cobra.Command{ + Use: "restore ", + Short: "Restore a volume or accessory backup (quiesced)", + Long: "Restores a backup under the app's fenced lock: the app's containers\n" + + "are stopped (and a generic accessory's container, for an accessory\n" + + "restore), the backup is promoted with the two-phase recovery\n" + + "discipline, the containers are started again and health-checked. On\n" + + "failure the recovery copy of the previous data is kept and named.\n" + + "Non-interactive runs must pass --yes.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runBackupRestore(flags, appName, accessory, args[0], &tf, yes) + }, + } + + cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + cmd.Flags().StringVar(&accessory, "accessory", "", "restore this accessory's backup instead of the app's volumes") + addBackupTargetFlags(cmd, &tf) + cmd.Flags().BoolVar(&yes, "yes", false, "proceed without interactive confirmation") + + return cmd +} + +func runBackupRestore(flags *Flags, appName, accessory, date string, tf *backupTargetFlags, yes bool) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() - executor, err := connectForApp(ctx, flags, appCfg) + appCfg, executor, err := resolveApp(ctx, flags, appName) if err != nil { return err } defer executor.Close() - client := backup.NewClient(executor, os.Stdout) - backups, err := client.ListBackups(ctx, appCfg.App, "volumes", s3Config(bucket, region, endpoint)) + cfg, err := resolveBackupS3(ctx, executor, tf) if err != nil { return err } - if flags.JSON { - type backupDTO struct { - Name string `json:"name"` + if err := backup.ValidateDate(date); err != nil { + return err + } + if accessory != "" { + if err := backup.ValidateBucket(accessory); err != nil { + return fmt.Errorf("invalid accessory name %q: %w", accessory, err) + } + } + + if !yes { + if !stdinIsTerminal() { + return fmt.Errorf("--yes is required to restore without an interactive terminal") } - result := make([]backupDTO, 0, len(backups)) - for _, item := range backups { - result = append(result, backupDTO{Name: item}) + if !confirm(os.Stdout, fmt.Sprintf("Restore %s backup %s for %s (stops the app while it runs)?", restoreKindLabel(accessory), date, appCfg.App)) { + return fmt.Errorf("aborted") } - return json.NewEncoder(os.Stdout).Encode(result) } - if len(backups) == 0 { - fmt.Println("No backups found") - return nil + // The fenced lock serializes the restore against deploys and every + // other mutating command; every stop/start below runs guarded by it. + lk, err := state.AcquireLockFenced(ctx, executor, appCfg.App) + if err != nil { + return err + } + lk.StartRenewal(executor) + defer state.ReleaseLockFenced(executor, lk, appCfg.App) + + progress := backupProgressOut(flags) + client := backup.NewClient(executor, progress) + lc := deploy.NewLifecycle(executor, progress) + + dto := backupRestoreDTO{ + MachineInterface: MachineInterface, + Host: executor.Host(), + App: appCfg.App, + ID: date, + Kind: backup.KindVolumes, + Accessory: accessory, + Stopped: []string{}, + Started: []string{}, + } + if accessory != "" { + dto.Kind = backup.KindAccessory } - for _, b := range backups { - fmt.Println(b) + // Quiesce: stop the app's running containers; a generic accessory's + // container too (engine restores run through the live engine; redis + // manages its own stop/start inside its restore script). + stopTimeout := appCfg.StopTimeout + if stopTimeout == 0 { + stopTimeout = 10 + } + stopped, qErr := lc.QuiesceApp(ctx, lk, appCfg.App, stopTimeout) + for _, c := range stopped { + dto.Stopped = append(dto.Stopped, c.Name) } + accessoryStopped := false + if qErr == nil && accessory != "" { + accessoryImage, _ := inspectAccessoryForRestore(ctx, client, appCfg.App, accessory, appCfg) + if backup.EngineOf(accessoryImage) == "generic" { + container := appCfg.App + "-" + accessory + if _, sErr := lk.Guarded(ctx, executor, fmt.Sprintf("docker stop --time %d %s", stopTimeout, ssh.ShellQuote(container))); sErr == nil { + accessoryStopped = true + dto.Stopped = append(dto.Stopped, container) + } else { + qErr = sErr + } + } + } + if qErr != nil { + // Quiesce failures leave the app down only when a stop failed + // midway — restart whatever QuiesceApp managed to stop. + return finishRestoreFailure(flags, ctx, executor, lc, lk, dto, accessoryStopped, stopped, qErr) + } + + // Restore, keeping ownership of the recovery directory until the + // post-restore health check also passes. + var recoveryDir string + quiescedNote := strings.Join(dto.Stopped, ", ") + if accessory != "" { + accessoryImage, env := inspectAccessoryForRestore(ctx, client, appCfg.App, accessory, appCfg) + recoveryDir, err = client.AccessoryRestoreKeep(ctx, appCfg.App, accessory, accessoryImage, date, env, cfg) + } else { + recoveryDir, err = client.RestoreVolumesKeep(ctx, appCfg.App, date, cfg, quiescedNote) + } + if err != nil { + return finishRestoreFailure(flags, ctx, executor, lc, lk, dto, accessoryStopped, stopped, err) + } + + // Resume: start what was stopped and health-check the app. + resume := lc.ResumeApp(ctx, lk, appCfg.App, stopped) + dto.Started = resume.Started + if dto.Started == nil { + dto.Started = []string{} + } + if accessoryStopped { + container := appCfg.App + "-" + accessory + if _, sErr := lk.Guarded(ctx, executor, "docker start "+ssh.ShellQuote(container)); sErr == nil { + dto.Started = append(dto.Started, container) + } else if resume.HealthErr == nil { + resume.HealthErr = sErr + } + } + + if resume.HealthErr != nil { + // The restored data is in place but did not come up healthy: keep + // the recovery copy so the operator can fall back. + dto.Health = "unhealthy" + dto.RecoveryDir = recoveryDir + dto.Error = fmt.Sprintf("health check after restore failed: %v — pre-restore data kept in %s", resume.HealthErr, recoveryDir) + recordAppEvent(ctx, executor, appCfg.App, events.KindBackupRestore, resume.HealthErr, restoreEventDetail(dto)) + if flags.JSON { + return errors.Join(resume.HealthErr, json.NewEncoder(os.Stdout).Encode(dto)) + } + return resume.HealthErr + } + dto.Health = resume.HealthStatus() + + if recoveryDir != "" { + executor.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(recoveryDir)) + } + dto.OK = true + recordAppEvent(ctx, executor, appCfg.App, events.KindBackupRestore, nil, restoreEventDetail(dto)) + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(dto) + } + fmt.Printf("Restore complete: %s (started %s)\n", date, strings.Join(dto.Started, ", ")) return nil } -func newBackupRestoreCmd(flags *Flags) *cobra.Command { +// inspectAccessoryForRestore resolves the accessory's image and env for a +// restore: the running container first (the resolved credentials only +// exist there, audit F68), the folder config only when it carries no +// unresolved references. +func inspectAccessoryForRestore(ctx context.Context, client *backup.Client, app, accessory string, appCfg *config.AppConfig) (string, map[string]string) { + if image, env, err := client.InspectAccessory(ctx, app, accessory); err == nil { + return image, env + } + if accCfg, ok := appCfg.Accessories[accessory]; ok { + if err := requireResolvedAccessoryEnv(accessory, accCfg.Env); err == nil { + return accCfg.Image, accCfg.Env + } + } + return "", nil +} + +// finishRestoreFailure restarts what was stopped (unless the failure left +// the data tree in an indeterminate state — recovery-incomplete — where +// serving from it is worse than staying down), reports, and records the +// event. +func finishRestoreFailure(flags *Flags, ctx context.Context, executor ssh.Executor, lc *deploy.Lifecycle, lk *state.Lock, dto backupRestoreDTO, accessoryStopped bool, stopped []docker.Container, rerr error) error { + if accessoryStopped { + container := dto.App + "-" + dto.Accessory + if _, err := lk.Guarded(ctx, executor, "docker start "+ssh.ShellQuote(container)); err == nil { + dto.Started = append(dto.Started, container) + } + } + if !backup.IsRecoveryIncomplete(rerr) { + resume := lc.ResumeApp(ctx, lk, dto.App, stopped) + dto.Started = append(dto.Started, resume.Started...) + } + dto.OK = false + dto.Error = rerr.Error() + recordAppEvent(ctx, executor, dto.App, events.KindBackupRestore, rerr, restoreEventDetail(dto)) + if flags.JSON { + return errors.Join(rerr, json.NewEncoder(os.Stdout).Encode(dto)) + } + return rerr +} + +func restoreEventDetail(dto backupRestoreDTO) map[string]any { + detail := map[string]any{"id": dto.ID, "kind": dto.Kind} + if dto.Accessory != "" { + detail["accessory"] = dto.Accessory + } + return detail +} + +func restoreKindLabel(accessory string) string { + if accessory != "" { + return "accessory " + accessory + } + return "volume" +} + +func newBackupPruneCmd(flags *Flags) *cobra.Command { var ( - bucket string - region string - endpoint string + appName string + tf backupTargetFlags + accessory string + keepLast int + maxAgeDays int ) cmd := &cobra.Command{ - Use: "restore ", - Short: "Restore a volume backup", - Args: cobra.ExactArgs(1), + Use: "prune", + Short: "Delete old backups per a retention policy", + Long: "Removes backups outside the retention policy.\nExample: teploy backup prune --keep-last 7\n teploy backup prune --max-age-days 30 --keep-last 3", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runBackupRestore(flags, args[0], bucket, region, endpoint) + return runBackupPrune(flags, appName, &tf, accessory, keepLast, maxAgeDays) }, } - cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") - cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") - cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") + cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + addBackupTargetFlags(cmd, &tf) + cmd.Flags().StringVar(&accessory, "accessory", "", "Prune an accessory's backups instead of volumes (accessory name)") + addRetentionFlags(cmd, &keepLast, &maxAgeDays) return cmd } +func runBackupPrune(flags *Flags, appName string, tf *backupTargetFlags, accessory string, keepLast, maxAgeDays int) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + appCfg, executor, err := resolveApp(ctx, flags, appName) + if err != nil { + return err + } + defer executor.Close() + + cfg, err := resolveBackupS3(ctx, executor, tf) + if err != nil { + return err + } + + policy := backup.RetentionPolicy{KeepLast: keepLast, MaxAgeDays: maxAgeDays} + if policy.IsZero() { + return fmt.Errorf("nothing to prune: set --keep-last and/or --max-age-days") + } + + prefix := "volumes" + if accessory != "" { + // Accessory name lands in an unquoted S3 path (ListBackups) — reuse the + // same safe-name check as buckets to keep it shell-safe. + if err := backup.ValidateBucket(accessory); err != nil { + return fmt.Errorf("invalid accessory name %q: %w", accessory, err) + } + prefix = "accessories/" + accessory + } + + client := backup.NewClient(executor, backupProgressOut(flags)) + deleted, err := client.PruneBackups(ctx, appCfg.App, prefix, cfg, policy) + if err != nil { + return err + } + + if len(deleted) == 0 { + fmt.Println("No backups pruned (all within retention policy)") + } else { + fmt.Printf("Pruned %d backup(s)\n", len(deleted)) + } + return nil +} + func newBackupScheduleCmd(flags *Flags) *cobra.Command { var ( - bucket string - region string - endpoint string - keepLast int + appName string + tf backupTargetFlags + keepLast int + maxAgeDays int ) cmd := &cobra.Command{ Use: "schedule ", Short: "Set up automated backups on a cron schedule", - Long: "Creates a cron job on the server to run backups automatically.\nExample: teploy backup schedule \"0 3 * * *\" --bucket my-backups --keep-last 7", - Args: cobra.ExactArgs(1), + Long: "Creates a cron job on the server that runs `teploy backup run` — the\n" + + "job carries no bucket and no credentials; both are read from the\n" + + "server's stored backup target at run time.\n" + + "Example: teploy backup schedule \"0 3 * * *\" --app myapp --keep-last 7", + Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runBackupSchedule(flags, args[0], bucket, region, endpoint, keepLast) + return runBackupSchedule(flags, args[0], appName, &tf, keepLast, maxAgeDays) }, } - cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") - cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") - cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") - cmd.Flags().IntVar(&keepLast, "keep-last", 0, "Keep only the N most recent backups on each run (0 = keep all)") + cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + addBackupTargetFlags(cmd, &tf) + addRetentionFlags(cmd, &keepLast, &maxAgeDays) return cmd } -func runBackupSchedule(flags *Flags, schedule, bucket, region, endpoint string, keepLast int) error { - appCfg, err := config.LoadApp(".") +func runBackupSchedule(flags *Flags, schedule, appName string, tf *backupTargetFlags, keepLast, maxAgeDays int) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + appCfg, executor, err := resolveApp(ctx, flags, appName) if err != nil { return err } + defer executor.Close() - if bucket == "" { - return fmt.Errorf("--bucket is required") - } - if err := backup.ValidateBucket(bucket); err != nil { - return err - } - if err := backup.ValidateRegion(region); err != nil { - return err - } if err := backup.ValidateSchedule(schedule); err != nil { return err } - if keepLast < 0 { - return fmt.Errorf("--keep-last must be >= 0") + if keepLast < 0 || maxAgeDays < 0 { + return fmt.Errorf("--keep-last and --max-age-days must be >= 0") } - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) - defer cancel() + target, err := resolveScheduleTarget(ctx, executor, tf) + if err != nil { + return err + } - executor, err := connectForApp(ctx, flags, appCfg) + // The scheduled job runs ON the server and needs a teploy binary that + // speaks `backup run`. + binPath, err := ensureScheduledBackupBinary(ctx, executor) if err != nil { return err } - defer executor.Close() + // Failure alerting stays a script beside the app; `backup run` invokes + // it on failure, so the cron line itself stays argument-free. webhook := firstWebhookURL(appCfg.Notifications) - - // The alert script goes up before the cron entry that calls it, so a - // schedule is never installed pointing at a script that is not there yet. signed := false if webhook != "" { secret := appCfg.Notifications.SigningSecret() @@ -321,21 +875,27 @@ func runBackupSchedule(flags *Flags, schedule, bucket, region, endpoint string, signed = secret != "" } - backupCmd := buildScheduledBackupCmd(appCfg.App, executor.Host(), bucket, region, endpoint, keepLast, webhook) + // The retention half of the schedule lives in the per-app store, read + // at run time. + policy, err := backup.ReadPolicy(ctx, executor, appCfg.App) + if err != nil { + return err + } + policy.Volumes = &backup.KindPolicy{Cron: schedule, KeepLast: keepLast, MaxAgeDays: maxAgeDays} + if err := backup.WritePolicy(ctx, executor, appCfg.App, policy); err != nil { + return err + } - client := backup.NewClient(executor, os.Stdout) - if err := client.SetSchedule(ctx, schedule, backupCmd, "teploy-backup:"+appCfg.App); err != nil { + client := backup.NewClient(executor, backupProgressOut(flags)) + if err := client.SetSchedule(ctx, schedule, buildScheduledBackupRunCmd(binPath, appCfg.App, ""), "teploy-backup:"+appCfg.App); err != nil { return err } fmt.Printf("Backup scheduled: %s\n", schedule) fmt.Printf(" App: %s\n", appCfg.App) - fmt.Printf(" Bucket: s3://%s/%s/volumes/\n", bucket, appCfg.App) - if endpoint != "" { - fmt.Printf(" Endpoint: %s\n", endpoint) - } - if keepLast > 0 { - fmt.Printf(" Retention: keep last %d\n", keepLast) + fmt.Printf(" Target: s3://%s (read at run time from the server's stored target)\n", backupTargetLabel(target)) + if keepLast > 0 || maxAgeDays > 0 { + fmt.Printf(" Retention: keep last %d, max age %dd\n", keepLast, maxAgeDays) } if webhook != "" { if signed { @@ -349,6 +909,71 @@ func runBackupSchedule(flags *Flags, schedule, bucket, region, endpoint string, return nil } +// storeTargetFromLegacyFlags persists the legacy flag values as the +// server's backup target (credentials from the local environment, when an +// endpoint is in play). Used only by the schedule bootstrap path. +func storeTargetFromLegacyFlags(ctx context.Context, executor ssh.Executor, tf *backupTargetFlags) (backup.Target, error) { + region := tf.region + if region == "" { + region = "us-east-1" + } + if err := backup.ValidateBucket(tf.bucket); err != nil { + return backup.Target{}, err + } + if err := backup.ValidateRegion(region); err != nil { + return backup.Target{}, err + } + if tf.endpoint != "" { + if err := backup.ValidateEndpoint(tf.endpoint); err != nil { + return backup.Target{}, err + } + } + if err := backup.ValidatePrefix(tf.prefix); err != nil { + return backup.Target{}, err + } + t := backup.Target{Bucket: tf.bucket, Region: region, Endpoint: tf.endpoint, Prefix: tf.prefix} + credsArmor := "" + if tf.endpoint != "" { + access := os.Getenv("TEPLOY_S3_ACCESS_KEY") + if access == "" { + access = os.Getenv("AWS_ACCESS_KEY_ID") + } + secret := os.Getenv("TEPLOY_S3_SECRET_KEY") + if secret == "" { + secret = os.Getenv("AWS_SECRET_ACCESS_KEY") + } + if access != "" && secret != "" { + if err := (secretstore.NewManager(executor)).EnsureAge(ctx); err != nil { + return backup.Target{}, err + } + armor, err := backup.EncryptCredentials(ctx, executor, access, secret) + if err != nil { + return backup.Target{}, err + } + credsArmor = armor + } + } + path, fallback, err := backup.WriteTarget(ctx, executor, t, credsArmor) + if err != nil { + return backup.Target{}, err + } + t.Path = path + t.HasCredentials = credsArmor != "" + fmt.Printf("Stored the server backup target s3://%s at %s\n", backupTargetLabel(t), path) + if fallback { + fmt.Println(" /etc/teploy is not writable by this user — the target lives in ~/.config/teploy instead") + } + return t, nil +} + +// backupTargetLabel renders a target as bucket[/prefix]. +func backupTargetLabel(t backup.Target) string { + if t.Prefix == "" { + return t.Bucket + } + return t.Bucket + "/" + t.Prefix +} + // firstWebhookURL returns the first usable webhook URL from the app's // notifications config (legacy single webhook, else the first webhook channel), // or "" if none. Only webhook channels work from the headless cron path — SMTP @@ -365,170 +990,44 @@ func firstWebhookURL(n config.NotificationsConfig) string { return "" } -// buildScheduledBackupCmd assembles the shell command cron runs for a scheduled -// backup: archive → upload → clean up, then optional keep-last retention, and -// (if a webhook is set) a failure alert wrapping the whole chain. -func buildScheduledBackupCmd(app, server, bucket, region, endpoint string, keepLast int, webhook string) string { - // A custom endpoint must reach every aws invocation (upload AND the - // retention loop): cron runs with the server's ambient AWS config, which - // points at real AWS — without the flag the scheduled backups upload to - // the wrong target. Same flag style as S3Config.AWS. - awsEndpoint := "" - if endpoint != "" { - awsEndpoint = " --endpoint-url " + ssh.ShellQuote(endpoint) - } - cmd := fmt.Sprintf( - "f=/tmp/%s-backup-$(date +%%Y%%m%%d-%%H%%M%%S).tar.gz && tar -czf \"$f\" -C /deployments/%s/volumes . && "+ - "aws s3 cp \"$f\" s3://%s/%s/volumes/ --region %s%s && "+ - "rm -f \"$f\"", - app, app, - bucket, app, region, awsEndpoint, - ) - - // The archive path is captured in $f rather than re-globbed: the old - // `aws s3 cp /tmp/-backup-*.tar.gz` matched ANY leftover archive - // from a previously interrupted run and uploaded (then deleted) it - // alongside the fresh one. - - // Bake keep-last retention into the same cron job: after the fresh upload, - // list the timestamped keys, keep the newest N, delete the rest. The names - // sort chronologically (20060102-150405), so `head -n -N` yields exactly the - // stale ones. No `%` in this clause, so no cron %-escaping concerns. Only - // keep-last is supported in the scheduled shell path; --max-age-days lives on - // `backup create`/`backup prune` (shell date math is fragile). - if keepLast > 0 { - cmd += fmt.Sprintf( - " && for f in $(aws s3 ls s3://%s/%s/volumes/ --region %s%s | sed 's/.* //' | grep -E '[0-9]{8}-[0-9]{6}' | sort | head -n -%d); do "+ - "[ -n \"$f\" ] && aws s3 rm s3://%s/%s/volumes/$f --region %s%s; done", - bucket, app, region, awsEndpoint, keepLast, - bucket, app, region, awsEndpoint, - ) - } - - // Failure alerting: a scheduled backup runs headless, so it can't use the CLI - // notifier (teploy.yml + its config aren't on the server). The alert lives in - // a script uploaded beside the app — see backup_alert.go for why it is a file - // rather than an inline curl: signing needs a timestamp, `date +%s` contains - // a `%`, and crontab treats `%` as a newline escape. `; false` preserves the - // non-zero exit so cron still logs the failure. - if webhook != "" { - cmd = "( " + cmd + " ) || { " + ssh.ShellQuote(backupAlertScriptPath(app)) + "; false; }" - } - - return cmd -} - -func runBackupRestore(flags *Flags, date, bucket, region, endpoint string) error { - appCfg, err := config.LoadApp(".") - if err != nil { - return err - } - - if bucket == "" { - return fmt.Errorf("--bucket is required") - } - if err := backup.ValidateBucket(bucket); err != nil { - return err - } - if err := backup.ValidateRegion(region); err != nil { - return err - } - if err := backup.ValidateDate(date); err != nil { - return err - } - - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) - defer cancel() - - executor, err := connectForApp(ctx, flags, appCfg) - if err != nil { - return err - } - defer executor.Close() - - client := backup.NewClient(executor, os.Stdout) - return client.RestoreVolumes(ctx, appCfg.App, date, s3Config(bucket, region, endpoint)) -} - -func newBackupPruneCmd(flags *Flags) *cobra.Command { - var ( - bucket string - region string - endpoint string - accessory string - keepLast int - maxAgeDays int - ) - +func newBackupUnscheduleCmd(flags *Flags) *cobra.Command { + var appName string cmd := &cobra.Command{ - Use: "prune", - Short: "Delete old backups per a retention policy", - Long: "Removes backups outside the retention policy.\nExample: teploy backup prune --bucket my-backups --keep-last 7\n teploy backup prune --bucket my-backups --max-age-days 30 --keep-last 3", + Use: "unschedule", + Short: "Remove the scheduled volume-backup cron entry", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runBackupPrune(flags, bucket, region, endpoint, accessory, keepLast, maxAgeDays) + return runBackupUnschedule(flags, appName) }, } - - cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") - cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") - cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") - cmd.Flags().StringVar(&accessory, "accessory", "", "Prune an accessory's backups instead of volumes (accessory name)") - addRetentionFlags(cmd, &keepLast, &maxAgeDays) - + cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") return cmd } -func runBackupPrune(flags *Flags, bucket, region, endpoint, accessory string, keepLast, maxAgeDays int) error { - appCfg, err := config.LoadApp(".") - if err != nil { - return err - } - - if bucket == "" { - return fmt.Errorf("--bucket is required") - } - if err := backup.ValidateBucket(bucket); err != nil { - return err - } - if err := backup.ValidateRegion(region); err != nil { - return err - } - - policy := backup.RetentionPolicy{KeepLast: keepLast, MaxAgeDays: maxAgeDays} - if policy.IsZero() { - return fmt.Errorf("nothing to prune: set --keep-last and/or --max-age-days") - } - - prefix := "volumes" - if accessory != "" { - // Accessory name lands in an unquoted S3 path (ListBackups) — reuse the - // same safe-name check as buckets to keep it shell-safe. - if err := backup.ValidateBucket(accessory); err != nil { - return fmt.Errorf("invalid accessory name %q: %w", accessory, err) - } - prefix = "accessories/" + accessory - } - +func runBackupUnschedule(flags *Flags, appName string) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() - executor, err := connectForApp(ctx, flags, appCfg) + appCfg, executor, err := resolveApp(ctx, flags, appName) if err != nil { return err } defer executor.Close() - client := backup.NewClient(executor, os.Stdout) - deleted, err := client.PruneBackups(ctx, appCfg.App, prefix, s3Config(bucket, region, endpoint), policy) + client := backup.NewClient(executor, backupProgressOut(flags)) + if err := client.RemoveSchedule(ctx, "teploy-backup:"+appCfg.App); err != nil { + return err + } + policy, err := backup.ReadPolicy(ctx, executor, appCfg.App) if err != nil { return err } - - if len(deleted) == 0 { - fmt.Println("No backups pruned (all within retention policy)") - } else { - fmt.Printf("Pruned %d backup(s)\n", len(deleted)) + if policy.Volumes != nil { + policy.Volumes = nil + if err := backup.WritePolicy(ctx, executor, appCfg.App, policy); err != nil { + return err + } } + fmt.Printf("Backup schedule removed for %s\n", appCfg.App) return nil } diff --git a/internal/cli/backup_cron_test.go b/internal/cli/backup_cron_test.go new file mode 100644 index 0000000..a0e1abc --- /dev/null +++ b/internal/cli/backup_cron_test.go @@ -0,0 +1,178 @@ +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/ssh" +) + +const testCrontab = `0 4 * * 0 /deployments/forgejo/scheduled-redeploy.sh >> /deployments/forgejo/scheduled-redeploy.log 2>&1 +30 4 * * * /usr/local/bin/teploy-nucleus-backup ship # [teploy-accessory-backup:ship:nucleus] +0 3 * * * f=/tmp/myapp-backup-$(date +%Y%m%d-%H%M%S).tar.gz && tar -czf "$f" -C /deployments/myapp/volumes . && aws s3 cp "$f" s3://old-bucket/myapp/volumes/ --region us-east-1 && rm -f "$f" && for x in $(aws s3 ls s3://old-bucket/myapp/volumes/ --region us-east-1 | sed 's/.* //' | sort | head -n -7); do [ -n "$x" ] && aws s3 rm s3://old-bucket/myapp/volumes/$x --region us-east-1; done # [teploy-backup:myapp] +*/10 * * * * echo $(date +%s) >> /tmp/heartbeat # unrelated line, pct char must survive +0 5 * * * TEPLOY_S3_ACCESS_KEY=ak TEPLOY_S3_SECRET_KEY=sk /usr/local/bin/teploy accessory backup db --local --app other --bucket b --region us-east-1 # [teploy-accessory-backup:other:db] +` + +// rewriteBackupCrons rewrites teploy-generated legacy lines to the run +// form, leaves hand-written commands (and unrelated lines, including +// their % characters) byte-identical, and migrates the retention baked +// into the legacy line. +func TestRewriteBackupCrons(t *testing.T) { + t.Setenv("TEPLOY_SERVER_BINARY", "/usr/local/bin/teploy") + mock := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "crontab -l", Output: testCrontab}, + ssh.MockCommand{Match: "'/usr/local/bin/teploy' backup run --help", Output: ""}, + ssh.MockCommand{Match: "crontab -", Output: ""}, + ssh.MockCommand{Match: "cat '/deployments/.teploy/apps/myapp", Output: ""}, + ssh.MockCommand{Match: "d=$(dirname", Output: ""}, + ) + ctx := context.Background() + rewritten, kept, err := rewriteBackupCrons(ctx, mock) + if err != nil { + t.Fatalf("rewriteBackupCrons: %v", err) + } + if len(rewritten) != 2 || rewritten[0] != "teploy-backup:myapp" || rewritten[1] != "teploy-accessory-backup:other:db" { + t.Fatalf("rewritten = %v", rewritten) + } + if len(kept) != 1 || kept[0] != "teploy-accessory-backup:ship:nucleus" { + t.Fatalf("kept = %v (the hand-written nucleus line must not be touched)", kept) + } + + // The new crontab is installed over stdin (after any policy write, so + // find the payload that carries cron markers). + var installed string + for _, in := range mock.Inputs { + if strings.Contains(in, "[teploy-backup:") || strings.Contains(in, "[teploy-accessory-backup:") { + installed = in + } + } + if installed == "" { + t.Fatalf("no crontab payload installed; inputs: %q", mock.Inputs) + } + if !strings.Contains(installed, "0 3 * * * /usr/local/bin/teploy backup run --app 'myapp' # [teploy-backup:myapp]") { + t.Errorf("volumes line not rewritten to the run form:\n%s", installed) + } + if !strings.Contains(installed, "0 5 * * * /usr/local/bin/teploy backup run --app 'other' --accessory 'db' # [teploy-accessory-backup:other:db]") { + t.Errorf("accessory line not rewritten to the run form:\n%s", installed) + } + for _, banned := range []string{"old-bucket", "TEPLOY_S3", "AWS_SECRET"} { + if strings.Contains(installed, banned) { + t.Errorf("rewritten crontab still embeds %s:\n%s", banned, installed) + } + } + if !strings.Contains(installed, "/usr/local/bin/teploy-nucleus-backup ship # [teploy-accessory-backup:ship:nucleus]") { + t.Errorf("hand-written line altered:\n%s", installed) + } + if !strings.Contains(installed, "echo $(date +%s) >> /tmp/heartbeat # unrelated line, pct char must survive") { + t.Errorf("unrelated line (with its percent character) not preserved byte-for-byte:\n%s", installed) + } + if !strings.Contains(installed, "/deployments/forgejo/scheduled-redeploy.sh") { + t.Errorf("unrelated first line dropped:\n%s", installed) + } +} + +// Without binary support the rewrite refuses rather than installing a +// cron line the server cannot run. +func TestRewriteBackupCronsNeedsBinarySupport(t *testing.T) { + t.Setenv("TEPLOY_SERVER_BINARY", "/opt/nope/teploy") + mock := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "crontab -l", Output: testCrontab}, + ssh.MockCommand{Match: "'/opt/nope/teploy' backup run --help", Err: errFakeCommand}, + ) + rewritten, kept, err := rewriteBackupCrons(context.Background(), mock) + if err != nil || len(rewritten) != 0 { + t.Fatalf("rewrote without binary support: %v %v %v", rewritten, kept, err) + } + if len(kept) != 3 { + t.Fatalf("kept = %v, want all three markers left alone", kept) + } +} + +var errFakeCommand = &fakeCmdError{} + +type fakeCmdError struct{} + +func (*fakeCmdError) Error() string { return "exit status 1" } + +// Doctor's backup-cron check fails on embedded credentials, warns on +// legacy teploy lines, and never echoes raw line text. +func TestDoctorBackupCronCheck(t *testing.T) { + ctx := context.Background() + mock := ssh.NewMockExecutor("srv", ssh.MockCommand{Match: "crontab -l", Output: testCrontab}) + check := doctorBackupCronCheck(ctx, mock) + if check.Result != doctorFail || !strings.Contains(check.Detail, "1 cron line") || !strings.Contains(check.Detail, "credentials") { + t.Fatalf("embedded credentials not failed: %+v", check) + } + for _, secret := range []string{"TEPLOY_S3_SECRET_KEY=sk", "AWS_SECRET"} { + if strings.Contains(check.Detail, secret) { + t.Errorf("check detail echoes raw line text: %+v", check) + } + } + + clean := ssh.NewMockExecutor("srv", ssh.MockCommand{Match: "crontab -l", Output: "30 3 * * * /usr/local/bin/teploy backup run --app 'myapp' # [teploy-backup:myapp]\n"}) + check = doctorBackupCronCheck(ctx, clean) + if check.Result != doctorOK { + t.Fatalf("clean crontab not ok: %+v", check) + } + + legacyOnly := strings.Split(testCrontab, "\n")[2] + legacy := ssh.NewMockExecutor("srv", ssh.MockCommand{Match: "crontab -l", Output: legacyOnly}) + check = doctorBackupCronCheck(ctx, legacy) + if check.Result != doctorWarn || !strings.Contains(check.Detail, "bucket") { + t.Fatalf("legacy line not warned: %+v", check) + } +} + +// Schedules join the cron markers with the stored retention. +func TestDiscoverBackupSchedules(t *testing.T) { + mock := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "crontab -l", Output: "30 3 * * * /usr/local/bin/teploy backup run --app 'myapp' # [teploy-backup:myapp]\n15 2 * * * /usr/local/bin/teploy backup run --app 'myapp' --accessory 'db' # [teploy-accessory-backup:myapp:db]\n"}, + ssh.MockCommand{Match: "cat '/deployments/.teploy/apps/myapp", Output: `{"volumes":{"cron":"30 3 * * *","keep_last":7},"accessories":{"db":{"cron":"15 2 * * *","keep_last":5,"max_age_days":30}}}`}, + ) + rows, err := discoverBackupSchedules(context.Background(), mock, "myapp") + if err != nil { + t.Fatalf("discoverBackupSchedules: %v", err) + } + if len(rows) != 2 { + t.Fatalf("rows = %+v", rows) + } + if rows[0].Kind != backup.KindVolumes || rows[0].KeepLast != 7 || rows[0].Form != backup.CronFormRun { + t.Errorf("volumes row: %+v", rows[0]) + } + if rows[1].Accessory != "db" || rows[1].KeepLast != 5 || rows[1].MaxAgeDays != 30 { + t.Errorf("accessory row: %+v", rows[1]) + } +} + +// A stored target backs the folderless commands; legacy flags win over it. +func TestResolveBackupS3(t *testing.T) { + ctx := context.Background() + stored := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"kept","region":"eu-west-1","endpoint":"http://127.0.0.1:9100","credentials":"AGE"}`}, + ) + cfg, err := resolveBackupS3(ctx, stored, &backupTargetFlags{}) + if err != nil { + t.Fatalf("resolveBackupS3: %v", err) + } + if cfg.Bucket != "kept" || cfg.Region != "eu-west-1" || cfg.Endpoint != "http://127.0.0.1:9100" || cfg.StoredTarget != "/etc/teploy/backup-target.json" { + t.Fatalf("stored target not applied: %+v", cfg) + } + + legacy, err := resolveBackupS3(ctx, stored, &backupTargetFlags{bucket: "override", endpoint: "http://127.0.0.1:9999"}) + if err != nil { + t.Fatalf("resolveBackupS3 legacy: %v", err) + } + if legacy.Bucket != "override" || legacy.StoredTarget != "" { + t.Fatalf("legacy flags must bypass the stored target: %+v", legacy) + } + + none := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent"}, + ) + if _, err := resolveBackupS3(ctx, none, &backupTargetFlags{}); err == nil || !strings.Contains(err.Error(), "no backup target") { + t.Fatalf("err = %v, want the no-target refusal", err) + } +} diff --git a/internal/cli/backup_run.go b/internal/cli/backup_run.go new file mode 100644 index 0000000..e980ca9 --- /dev/null +++ b/internal/cli/backup_run.go @@ -0,0 +1,199 @@ +package cli + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/events" + "github.com/useteploy/teploy/internal/ssh" +) + +// `teploy backup run` — the cron entrypoint (D3). It runs ON the server +// (like `autodeploy redeploy` and `accessory backup --local`): docker, aws +// and the stored backup target are all local, so there is no SSH hop and +// no teploy.yml. Cron lines carry only the app (and accessory); the +// target, retention and the failure-alert script are read at run time. + +func newBackupRunCmd(flags *Flags) *cobra.Command { + var ( + appName string + accessory string + ) + cmd := &cobra.Command{ + Use: "run", + Short: "Run a scheduled backup on the server (the cron entrypoint)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runBackupRun(flags, appName, accessory) + }, + } + cmd.Flags().StringVar(&appName, "app", "", "app name (required)") + cmd.Flags().StringVar(&accessory, "accessory", "", "back up this accessory instead of the app's volumes") + return cmd +} + +func runBackupRun(flags *Flags, appName, accessory string) error { + if err := config.ValidateName(appName); err != nil { + return err + } + if accessory != "" { + if err := backup.ValidateBucket(accessory); err != nil { + return fmt.Errorf("invalid accessory name %q: %w", accessory, err) + } + } + + // Runs on the server itself: no SSH hop, local docker and aws. + executor := ssh.NewLocalExecutor() + defer executor.Close() + return runBackupRunOn(context.Background(), executor, appName, accessory) +} + +// runBackupRunOn is runBackupRun against a given executor (the seam tests +// use; production passes the local executor). +func runBackupRunOn(ctx context.Context, executor ssh.Executor, appName, accessory string) error { + + target, err := backup.ReadTarget(ctx, executor) + if err != nil { + return err + } + if !TargetConfigured(target) { + return fmt.Errorf("no backup target is configured on this server — run `teploy server backup-target set` from a machine that manages it") + } + cfg := TargetS3(target) + + client := backup.NewClient(executor, os.Stdout) + policy, err := backup.ReadPolicy(ctx, executor, appName) + if err != nil { + return err + } + + var res backup.BackupResult + var kind string + if accessory != "" { + image, env, inspErr := client.InspectAccessory(ctx, appName, accessory) + if inspErr != nil { + err = fmt.Errorf("inspecting accessory %s: %w", accessory, inspErr) + kind = backup.KindAccessory + } else { + res, err = client.AccessoryBackup(ctx, appName, accessory, image, env, cfg) + kind = backup.KindAccessory + } + } else { + res, err = client.BackupVolumes(ctx, appName, cfg) + kind = backup.KindVolumes + } + + // Retention from the per-app store (written by `backup schedule` / + // `accessory backup --schedule`), never baked into the cron line. + var pruned int + if err == nil { + var kp *backup.KindPolicy + if accessory == "" { + kp = policy.Volumes + } else if policy.Accessories != nil { + kp = policy.Accessories[accessory] + } + if kp != nil { + rp := backup.RetentionPolicy{KeepLast: kp.KeepLast, MaxAgeDays: kp.MaxAgeDays} + if !rp.IsZero() { + prefix := "volumes" + if accessory != "" { + prefix = "accessories/" + accessory + } + var deleted []string + deleted, pruneErr := client.PruneBackups(ctx, appName, prefix, cfg, rp) + pruned = len(deleted) + if pruneErr != nil { + fmt.Fprintf(os.Stderr, "Warning: prune failed: %v\n", pruneErr) + } + } + } + } + + detail := map[string]any{"id": res.ID, "kind": kind} + if accessory != "" { + detail["accessory"] = accessory + } + events.Record(ctx, executor, events.Event{App: appName, Kind: events.KindBackupRun, Outcome: events.Outcome(err), Detail: detail}) + + if err != nil { + // The failure alert (uploaded by `backup schedule` when the app's + // config has a webhook) fires from here, keeping the cron line a + // single argument-free command. + runBackupAlertIfPresent(ctx, appName) + return err + } + fmt.Printf("Backup %s complete (%s, %d bytes", res.ID, kind, res.SizeBytes) + if pruned > 0 { + fmt.Printf(", pruned %d", pruned) + } + fmt.Println(")") + return nil +} + +// runBackupAlertIfPresent executes the app's uploaded failure-alert script +// when one exists. Its own failures are reported but never mask the backup +// error that triggered it. +func runBackupAlertIfPresent(ctx context.Context, app string) { + script := backupAlertScriptPath(app) + exec := ssh.NewLocalExecutor() + defer exec.Close() + res := ssh.RunDetailed(ctx, exec, "if [ -x "+ssh.ShellQuote(script)+" ]; then "+ssh.ShellQuote(script)+"; fi") + if res.Failed() { + fmt.Fprintf(os.Stderr, "Warning: the backup alert script failed: %s\n", res.ExitErrorText()) + } +} + +// buildScheduledBackupRunCmd assembles the command every teploy-managed +// backup cron line runs: the binary by absolute path, the app, nothing +// else. Bucket, credentials and retention are all read at run time. +func buildScheduledBackupRunCmd(binPath, app, accessory string) string { + cmd := binPath + " backup run --app " + ssh.ShellQuote(app) + if accessory != "" { + cmd += " --accessory " + ssh.ShellQuote(accessory) + } + return cmd +} + +// scheduledTeployPath is the binary cron lines invoke. TEPLOY_SERVER_BINARY +// overrides it for testing against a pre-release build (the release +// pipeline must publish `backup run` before the default path is usable). +func scheduledTeployPath() string { + if p := strings.TrimSpace(os.Getenv("TEPLOY_SERVER_BINARY")); p != "" { + return p + } + return serverTeployPath +} + +// ensureScheduledBackupBinary makes sure the server has a teploy binary +// that speaks `backup run`, returning the path cron lines should invoke. +// An existing binary that already supports it is left untouched (no +// reinstall); otherwise the release pipeline installs the latest one and +// the candidate is capability-checked before publication. +func ensureScheduledBackupBinary(ctx context.Context, executor ssh.Executor) (string, error) { + bin := scheduledTeployPath() + if serverBinarySupportsBackupRun(ctx, executor, bin) { + return bin, nil + } + if bin != serverTeployPath { + return "", fmt.Errorf("%s does not support `backup run` (check TEPLOY_SERVER_BINARY)", bin) + } + if _, err := deployTeployBinaryToServer(ctx, executor, serverTeployPath, "backup run"); err != nil { + return "", fmt.Errorf("the server's teploy binary does not support `backup run` and installing one failed: %w", err) + } + if !serverBinarySupportsBackupRun(ctx, executor, bin) { + return "", fmt.Errorf("the installed teploy binary still does not support `backup run` — the latest release may predate it") + } + return bin, nil +} + +// serverBinarySupportsBackupRun probes the binary's own help. +func serverBinarySupportsBackupRun(ctx context.Context, executor ssh.Executor, bin string) bool { + res := ssh.RunDetailed(ctx, executor, ssh.ShellQuote(bin)+" backup run --help >/dev/null 2>&1") + return !res.Failed() +} diff --git a/internal/cli/backup_schedule_test.go b/internal/cli/backup_schedule_test.go index 8e5875b..58f38ba 100644 --- a/internal/cli/backup_schedule_test.go +++ b/internal/cli/backup_schedule_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/config" ) @@ -26,157 +27,90 @@ func TestFirstWebhookURL(t *testing.T) { } } -func TestBuildScheduledBackupCmd(t *testing.T) { - // Base: archive + upload + cleanup, no retention, no alert. The exact - // archive path is captured in $f (teploy-cli-12): re-globbing - // /tmp/-backup-*.tar.gz uploaded and deleted leftovers from - // interrupted runs alongside the fresh archive. - base := buildScheduledBackupCmd("myapp", "1.2.3.4", "my-bucket", "us-east-1", "", 0, "") +// The generated cron line must be the bare run form (D3): absolute binary +// path, the app, nothing else. The bucket and the credentials live in the +// server's stored target and are read at run time, so a leaked crontab +// leaks nothing, and a target change never rewrites cron. +func TestBuildScheduledBackupRunCmd(t *testing.T) { + vol := buildScheduledBackupRunCmd("/usr/local/bin/teploy", "myapp", "") for _, want := range []string{ - `f=/tmp/myapp-backup-$(date +`, - `tar -czf "$f" -C /deployments/myapp/volumes .`, - `aws s3 cp "$f" s3://my-bucket/myapp/volumes/ --region us-east-1`, - `rm -f "$f"`, + "/usr/local/bin/teploy backup run --app 'myapp'", } { - if !strings.Contains(base, want) { - t.Errorf("base cmd missing %q:\n%s", want, base) + if !strings.Contains(vol, want) { + t.Errorf("volumes cmd missing %q: %s", want, vol) } } - if strings.Contains(base, "-backup-*") { - t.Errorf("upload/cleanup must target only the captured archive, not a glob:\n%s", base) - } - if strings.Contains(base, "head -n -") || strings.Contains(base, "curl") { - t.Errorf("base cmd should have no retention/alert:\n%s", base) - } - - // Keep-last bakes a prune clause. - withKeep := buildScheduledBackupCmd("myapp", "1.2.3.4", "my-bucket", "us-east-1", "", 7, "") - if !strings.Contains(withKeep, "head -n -7") { - t.Errorf("keep-last prune clause missing:\n%s", withKeep) - } - - // teploy-cli-11: --endpoint must reach EVERY aws invocation — the - // upload AND the keep-last retention loop. The scheduled job runs with - // the server's ambient AWS config, which points at real AWS; without - // the flag, scheduled backups silently upload to the wrong target. - withEndpoint := buildScheduledBackupCmd("myapp", "1.2.3.4", "my-bucket", "us-east-1", "http://minio:9000", 2, "") - if n := strings.Count(withEndpoint, " --endpoint-url 'http://minio:9000'"); n != 3 { - t.Errorf("endpoint flag must be on all 3 aws invocations (cp, ls, rm), got %d:\n%s", n, withEndpoint) - } - if strings.Contains(base, "--endpoint-url") { - t.Errorf("no endpoint configured, so no endpoint flag expected:\n%s", base) - } - - // Webhook wraps the chain in a failure alert. The alert itself is now a - // script on the server rather than an inline curl (see backup_alert.go): the - // payload and the request moved into buildBackupAlertScript, which is what - // makes signing possible at all, so those assertions live in - // TestBuildBackupAlertScript. What the cron line still has to get right is - // the wrapping and the exit code. - withHook := buildScheduledBackupCmd("myapp", "1.2.3.4", "my-bucket", "us-east-1", "", 7, "https://hooks.example.com/x") - if !strings.HasPrefix(withHook, "( ") || !strings.Contains(withHook, ") || {") { - t.Errorf("alert wrapping missing:\n%s", withHook) - } - if !strings.Contains(withHook, backupAlertScriptPath("myapp")) { - t.Errorf("alert clause does not invoke the alert script:\n%s", withHook) - } - // `; false` preserves the non-zero exit so cron still logs the failure. - if !strings.Contains(withHook, "false; }") { - t.Errorf("alert clause must preserve the non-zero exit:\n%s", withHook) + for _, banned := range []string{"bucket", "TEPLOY_S3", "AWS_SECRET", "AWS_ACCESS", "region"} { + if strings.Contains(vol, banned) { + t.Errorf("volumes cmd must carry no bucket/credentials/env (%s): %s", banned, vol) + } } - // The alert clause must still be %-free: it is the part of the crontab line - // that used to trip cron's %-escaping, and the reason the body moved to a - // file is that a file has no such restriction. - alertPart := withHook[strings.Index(withHook, ") || {"):] - if strings.Contains(alertPart, "%") { - t.Errorf("alert clause must contain no '%%':\n%s", alertPart) + acc := buildScheduledBackupRunCmd("/usr/local/bin/teploy", "myapp", "db") + if !strings.Contains(acc, "--accessory 'db'") { + t.Errorf("accessory cmd missing the accessory flag: %s", acc) } } -func TestBuildBackupAlertScript(t *testing.T) { - const ( - app = "akiroo-lite" - server = "deploy-home2" - url = "https://akiroo.example.com/api/webhooks/teploy_cli/dlbs" - ) - - signed := buildBackupAlertScript(app, server, url, "s3cr3t") - - // The payload still has to match notify.Payload, since the receiver parses - // the same shape from every teploy delivery. - for _, want := range []string{ - `"app":"akiroo-lite"`, `"server":"deploy-home2"`, - `"type":"backup"`, `"success":false`, - } { - if !strings.Contains(signed, want) { - t.Errorf("script payload missing %q:\n%s", want, signed) - } +// Marker classification drives both the `backup-target set` rewrite and +// the doctor check: teploy-generated legacy lines rewrite, hand-written +// commands under a teploy marker (the nucleus-backup pattern) must be +// left alone, and the run form is already current. +func TestFindCronMarkerForms(t *testing.T) { + cases := []struct { + name string + line string + form string + kind string + app string + acc string + keep int + }{ + { + name: "run volumes", + line: "30 3 * * * /usr/local/bin/teploy backup run --app myapp # [teploy-backup:myapp]", + form: backup.CronFormRun, kind: "volumes", app: "myapp", + }, + { + name: "run accessory", + line: "15 2 * * * /usr/local/bin/teploy backup run --app myapp --accessory db # [teploy-accessory-backup:myapp:db]", + form: backup.CronFormRun, kind: "accessory", app: "myapp", acc: "db", + }, + { + name: "legacy volumes shell chain with retention", + line: "0 3 * * * f=/tmp/myapp-backup-$(date +%Y%m%d-%H%M%S).tar.gz && tar -czf \"$f\" -C /deployments/myapp/volumes . && aws s3 cp \"$f\" s3://bucket/myapp/volumes/ --region us-east-1 && rm -f \"$f\" && for x in $(aws s3 ls s3://bucket/myapp/volumes/ | head -n -7); do :; done # [teploy-backup:myapp]", + form: backup.CronFormLegacy, kind: "volumes", app: "myapp", keep: 7, + }, + { + name: "legacy accessory invocation with embedded creds", + line: "30 4 * * * TEPLOY_S3_ACCESS_KEY=ak TEPLOY_S3_SECRET_KEY=sk /usr/local/bin/teploy accessory backup nucleus --local --app ship --bucket b --region us-east-1 # [teploy-accessory-backup:ship:nucleus]", + form: backup.CronFormLegacy, kind: "accessory", app: "ship", acc: "nucleus", + }, + { + name: "hand-written script under a marker", + line: "30 4 * * * /usr/local/bin/teploy-nucleus-backup ship # [teploy-accessory-backup:ship:nucleus]", + form: backup.CronFormCustom, kind: "accessory", app: "ship", acc: "nucleus", + }, } - // Signed with the same construction as internal/notify/sign.go: HMAC over - // timestamp + "." + body, in the same two headers. - for _, want := range []string{ - "date +%s", - `printf '%s.%s' "$TS" "$BODY"`, - "openssl dgst -sha256 -hmac", - "X-Teploy-Timestamp: $TS", - "X-Teploy-Signature: sha256=$SIG", - } { - if !strings.Contains(signed, want) { - t.Errorf("signed script missing %q:\n%s", want, signed) + for _, tc := range cases { + m := backup.FindCronMarker(tc.line) + if m == nil { + t.Fatalf("%s: marker not found", tc.name) } - } - // It is a script, so % is allowed here — that is the entire reason this - // moved out of the crontab line. - if !strings.HasPrefix(signed, "#!/bin/sh\n") { - t.Error("script has no shebang, so cron cannot execute it") - } - - // No secret: still delivers, unsigned, exactly as before. - unsigned := buildBackupAlertScript(app, server, url, "") - if !strings.Contains(unsigned, "curl -sf -m 10 -X POST") { - t.Errorf("unsigned script does not deliver:\n%s", unsigned) - } - if strings.Contains(unsigned, "SECRET=''") == false { - t.Errorf("unsigned script should carry an empty SECRET:\n%s", unsigned) - } - - // An unsigned fallback must exist even in the signed script: if openssl is - // missing on the box, a delivery that arrives and gets rejected tells the - // operator something, and silence tells them nothing. - if strings.Count(signed, "curl -sf -m 10 -X POST") < 2 { - t.Errorf("signed script has no unsigned fallback path:\n%s", signed) - } - if !strings.Contains(signed, "command -v openssl") { - t.Errorf("signed script does not check openssl is present:\n%s", signed) - } -} - -// Every interpolated value is shell-quoted. app, server and the URL all come -// from configuration, and this file runs as root from cron — a value able to -// close a quote and append a command would be a root shell. -func TestBackupAlertScriptQuotesHostileValues(t *testing.T) { - script := buildBackupAlertScript( - "app'; touch /tmp/pwned; '", - "srv'; id; '", - "https://x/'; whoami; '", - "sec'; echo no; '", - ) - for _, injected := range []string{"touch /tmp/pwned", "; id;", "whoami", "echo no"} { - // The text may appear inside a quoted literal, but never as a command: - // an unescaped quote followed by it is what makes it executable. - if strings.Contains(script, "' "+injected) || strings.Contains(script, "';"+injected) { - t.Errorf("value escaped its quoting and became a command (%q):\n%s", injected, script) + if m.Form != tc.form || m.Kind != tc.kind || m.App != tc.app || m.Accessory != tc.acc { + t.Errorf("%s: got form=%s kind=%s app=%s acc=%s, want %s/%s/%s/%s", tc.name, m.Form, m.Kind, m.App, m.Accessory, tc.form, tc.kind, tc.app, tc.acc) + } + if m.KeepLast != tc.keep { + t.Errorf("%s: keep-last = %d, want %d", tc.name, m.KeepLast, tc.keep) + } + if m.Cron == "" || strings.Contains(m.Cron, "teploy") { + t.Errorf("%s: cron fields wrong: %q", tc.name, m.Cron) } } - // ShellQuote doubles single quotes; the payload's own quotes must survive. - if !strings.Contains(script, "BODY='") || !strings.Contains(script, "URL='") || !strings.Contains(script, "SECRET='") { - t.Errorf("script values are not single-quoted:\n%s", script) - } -} -func TestBackupAlertScriptPath(t *testing.T) { - // Under the app's own deployment directory, so removing the app removes it. - if got := backupAlertScriptPath("myapp"); got != "/deployments/myapp/backup-alert.sh" { - t.Errorf("backupAlertScriptPath = %q", got) + if m := backup.FindCronMarker("17 */6 * * * /usr/local/bin/infra-maintenance.sh"); m != nil { + t.Errorf("unrelated cron line classified as a backup marker: %+v", m) + } + if m := backup.FindCronMarker("30 4 * * * /usr/local/bin/teploy-nucleus-backup ship"); m != nil { + t.Errorf("marker-less teploy line classified: %+v", m) } } diff --git a/internal/cli/backup_target.go b/internal/cli/backup_target.go new file mode 100644 index 0000000..bd8d833 --- /dev/null +++ b/internal/cli/backup_target.go @@ -0,0 +1,537 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/signal" + "regexp" + "sort" + "strings" + + "github.com/useteploy/teploy/internal/config" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/events" + "github.com/useteploy/teploy/internal/secret" + "github.com/useteploy/teploy/internal/ssh" + "golang.org/x/term" +) + +// `teploy server backup-target` (D3): one backup target per server, stored +// outside /deployments and read at run time by every backup command, cron +// line and DR invocation. + +func newServerBackupTargetCmd(flags *Flags) *cobra.Command { + cmd := &cobra.Command{ + Use: "backup-target", + Short: "Manage the server's stored backup target", + Long: "One S3 target per server: bucket, region, endpoint and an optional\n" + + "key prefix, stored at /etc/teploy/backup-target.json (0600, outside\n" + + "/deployments). Credentials, when given, are age-encrypted with the\n" + + "server's own identity and are used whether or not an endpoint is set;\n" + + "without them the server's ambient AWS credential chain applies.\n\n" + + "A non-root user who cannot write /etc/teploy gets\n" + + "~/.config/teploy/backup-target.json instead, and the command says so.\n\n" + + "Setting a target rewrites teploy-managed backup cron lines to call\n" + + "only `teploy backup run` (no bucket, no credentials) — hand-written\n" + + "lines under a teploy marker are left alone and reported.", + } + cmd.AddCommand(newServerBackupTargetSetCmd(flags)) + cmd.AddCommand(newServerBackupTargetShowCmd(flags)) + cmd.AddCommand(newServerBackupTargetUnsetCmd(flags)) + return cmd +} + +// backupTargetDTO is the `server backup-target show --json` envelope +// (backup-target-v1). It never carries secret material — only whether any +// is stored. +type backupTargetDTO struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + Configured bool `json:"configured"` + Bucket string `json:"bucket,omitempty"` + Region string `json:"region,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Prefix string `json:"prefix,omitempty"` + HasCredentials bool `json:"has_credentials"` + Path string `json:"path,omitempty"` +} + +func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { + var ( + bucket string + region string + endpoint string + prefix string + noCredentials bool + ) + cmd := &cobra.Command{ + Use: "set", + Short: "Store the server's backup target (credentials on stdin)", + Long: "Stores the target and (unless --no-credentials) the S3 credentials,\n" + + "age-encrypted, on the server. Credentials are read from stdin as TWO\n" + + "lines: the access key, then the secret key. With a terminal, both are\n" + + "prompted with echo off.\n\n" + + "Example:\n" + + " printf 'ACCESS\\nSECRET\\n' | teploy server backup-target set --host prod --bucket b --endpoint https://s3.example.com", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runServerBackupTargetSet(flags, bucket, region, endpoint, prefix, noCredentials) + }, + } + cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") + cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") + cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2)") + cmd.Flags().StringVar(&prefix, "prefix", "", "key prefix under the bucket (optional)") + cmd.Flags().BoolVar(&noCredentials, "no-credentials", false, "store no credentials — use the server's ambient AWS credential chain") + return cmd +} + +func runServerBackupTargetSet(flags *Flags, bucket, region, endpoint, prefix string, noCredentials bool) error { + if bucket == "" { + return fmt.Errorf("--bucket is required") + } + if err := backup.ValidateBucket(bucket); err != nil { + return err + } + if err := backup.ValidateRegion(region); err != nil { + return err + } + if endpoint != "" { + if err := backup.ValidateEndpoint(endpoint); err != nil { + return err + } + } + if err := backup.ValidatePrefix(prefix); err != nil { + return err + } + + accessKey, secretKey, err := readBackupTargetCredentials(os.Stdin, noCredentials) + if err != nil { + return err + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return err + } + if !flags.JSON { + fmt.Printf("Connecting to %s@%s...\n", user, host) + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer executor.Close() + + // Encrypt first (all failure modes leave the old target untouched). + credsArmor := "" + if accessKey != "" { + if err := (secret.NewManager(executor)).EnsureAge(ctx); err != nil { + return err + } + credsArmor, err = backup.EncryptCredentials(ctx, executor, accessKey, secretKey) + if err != nil { + return err + } + } + t := backup.Target{Bucket: bucket, Region: region, Endpoint: endpoint, Prefix: prefix, HasCredentials: credsArmor != ""} + path, fallback, err := backup.WriteTarget(ctx, executor, t, credsArmor) + if err != nil { + return err + } + t.Path = path + + fmt.Printf("Backup target stored: s3://%s at %s\n", backupTargetLabel(t), path) + fmt.Printf(" Credentials: %s\n", credentialMode(t)) + if fallback { + fmt.Println(" /etc/teploy is not writable by this user — the target lives in ~/.config/teploy instead") + } + + // Best-effort reachability check through the exact run-time path + // (stored creds decrypt + aws). A failure warns; it does not refuse the + // set (the endpoint may be reachable only from cron's network, etc). + checkReachability(ctx, executor, t) + + rewritten, kept, err := rewriteBackupCrons(ctx, executor) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not rewrite backup cron lines: %v\n", err) + } else { + for _, m := range rewritten { + fmt.Printf(" Rewrote cron marker %s to `backup run` (bucket/credentials no longer embedded)\n", m) + } + for _, m := range kept { + fmt.Printf(" Cron marker %s was left alone (hand-written command) — check it still matches this target\n", m) + } + } + + recordBackupTargetEvent(ctx, executor, nil) + return nil +} + +func credentialMode(t backup.Target) string { + if t.HasCredentials { + return "stored, age-encrypted on the server" + } + return "none stored — the server's ambient AWS credential chain is used" +} + +func checkReachability(ctx context.Context, executor ssh.Executor, t backup.Target) { + cfg := TargetS3(t) + client := backup.NewClient(executor, io.Discard) + if err := client.EnsureAWSCLIPublic(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not verify the target: %v\n", err) + return + } + res := ssh.RunDetailed(ctx, executor, cfg.AWS("s3 ls s3://"+cfg.Bucket+"/")) + if res.Failed() { + fmt.Fprintf(os.Stderr, "Warning: the stored target did not answer `aws s3 ls` (%s) — check the bucket, endpoint and credentials\n", oneLine(res.ExitErrorText())) + return + } + fmt.Println(" Reachability: aws s3 ls answered") +} + +func oneLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexAny(s, "\r\n"); i >= 0 { + s = s[:i] + } + if len(s) > 160 { + s = s[:160] + "..." + } + return s +} + +// readBackupTargetCredentials reads the two-line credential pair. With a +// terminal both values are prompted (echo off); otherwise stdin must carry +// exactly two lines. An empty access key with --no-credentials (or empty +// stdin lines declined interactively) stores no credentials. +func readBackupTargetCredentials(r io.Reader, noCredentials bool) (accessKey, secretKey string, err error) { + if noCredentials { + return "", "", nil + } + if stdinIsTerminal() { + fmt.Print("S3 access key (empty for none): ") + accessKey, err = readHiddenLine() + if err != nil { + return "", "", err + } + if accessKey == "" { + return "", "", nil + } + fmt.Print("S3 secret key: ") + secretKey, err = readHiddenLine() + if err != nil { + return "", "", err + } + fmt.Println() + if secretKey == "" { + return "", "", fmt.Errorf("a secret key is required when an access key is given") + } + return accessKey, secretKey, nil + } + lines, err := readTwoLines(r) + if err != nil { + return "", "", err + } + if lines[0] == "" && lines[1] == "" { + return "", "", fmt.Errorf("credentials are required on stdin (two lines: access key, then secret key) — or pass --no-credentials to use the server's ambient AWS chain") + } + if lines[0] == "" || lines[1] == "" { + return "", "", fmt.Errorf("stdin must carry BOTH the access key and the secret key, one per line") + } + return lines[0], lines[1], nil +} + +// readHiddenLine reads one line from the terminal with echo off. +func readHiddenLine() (string, error) { + b, err := term.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + return "", err + } + return string(b), nil +} + +func readTwoLines(r io.Reader) ([2]string, error) { + var lines [2]string + buf := make([]byte, 0, 512) + var got [2]string + idx := 0 + ch := make([]byte, 1) + for n, err := r.Read(ch); err == nil && n > 0; n, err = r.Read(ch) { + if ch[0] == '\n' { + got[idx] = strings.TrimSuffix(string(buf), "\r") + buf = buf[:0] + idx++ + if idx == 2 { + return got, nil + } + continue + } + if len(buf) < 512 { + buf = append(buf, ch[0]) + } + } + if idx == 1 && len(buf) > 0 { + got[1] = strings.TrimSuffix(string(buf), "\r") + return got, nil + } + if idx == 0 && len(buf) > 0 { + got[0] = strings.TrimSuffix(string(buf), "\r") + return got, nil + } + return lines, fmt.Errorf("stdin must carry two lines: the access key, then the secret key") +} + +func newServerBackupTargetShowCmd(flags *Flags) *cobra.Command { + cmd := &cobra.Command{ + Use: "show", + Short: "Show the stored backup target (never credentials)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runServerBackupTargetShow(flags) + }, + } + return cmd +} + +func runServerBackupTargetShow(flags *Flags) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return err + } + if !flags.JSON { + fmt.Printf("Connecting to %s@%s...\n", user, host) + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer executor.Close() + + target, err := backup.ReadTarget(ctx, executor) + if err != nil { + return err + } + if flags.JSON { + return writeBackupTargetJSON(os.Stdout, executor.Host(), target) + } + if !TargetConfigured(target) { + fmt.Println("No backup target configured — set one with `teploy server backup-target set`") + return nil + } + fmt.Printf("Backup target: s3://%s\n", backupTargetLabel(target)) + if target.Endpoint != "" { + fmt.Printf(" Endpoint: %s\n", target.Endpoint) + } + fmt.Printf(" Region: %s\n", target.Region) + fmt.Printf(" Credentials: %s\n", credentialMode(target)) + fmt.Printf(" Stored at: %s\n", target.Path) + return nil +} + +func writeBackupTargetJSON(w io.Writer, host string, target backup.Target) error { + dto := backupTargetDTO{ + MachineInterface: MachineInterface, + Host: host, + HasCredentials: target.HasCredentials, + Path: target.Path, + } + if TargetConfigured(target) { + dto.Configured = true + dto.Bucket = target.Bucket + dto.Region = target.Region + dto.Endpoint = target.Endpoint + dto.Prefix = target.Prefix + } + return json.NewEncoder(w).Encode(dto) +} + +func newServerBackupTargetUnsetCmd(flags *Flags) *cobra.Command { + cmd := &cobra.Command{ + Use: "unset", + Short: "Remove the stored backup target", + Long: "Removes the stored target (both credentials and settings). Scheduled\n" + + "backup jobs will fail at run time until a new target is set — remove\n" + + "their cron entries first if you are retiring backups.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runServerBackupTargetUnset(flags) + }, + } + return cmd +} + +func runServerBackupTargetUnset(flags *Flags) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return err + } + if !flags.JSON { + fmt.Printf("Connecting to %s@%s...\n", user, host) + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer executor.Close() + + removed, err := backup.UnsetTarget(ctx, executor) + if err != nil { + return err + } + if len(removed) == 0 { + fmt.Println("No backup target configured") + return nil + } + for _, p := range removed { + fmt.Printf("Removed %s\n", p) + } + fmt.Println("Scheduled backup jobs will fail at run time until a new target is set.") + recordBackupTargetEvent(ctx, executor, nil) + return nil +} + +// configResolveServerForTarget resolves the target server for the +// backup-target family: --host names (or overrides) the server. +func configResolveServerForTarget(flags *Flags) (host, user, key string, err error) { + if flags.Host == "" { + return "", "", "", fmt.Errorf("--host is required (the backup target is per server)") + } + return config.ResolveServer(flags.Host, flags.Host, flags.User, flags.Key) +} + +// rewriteBackupCrons rewrites teploy-GENERATED backup cron lines (the +// pre-D3 forms, which embed the bucket — and for custom endpoints, the +// credentials) to the `backup run` form. Lines under a teploy marker whose +// command is not one of the generated shapes are hand-written and are left +// untouched (reported instead). The whole crontab is reinstalled over +// stdin, so unrelated lines — including their % characters — round-trip +// byte-for-byte. +func rewriteBackupCrons(ctx context.Context, executor ssh.Executor) (rewritten, kept []string, err error) { + lines, err := backup.ReadCrontab(ctx, executor) + if err != nil { + return nil, nil, err + } + bin := scheduledTeployPath() + supported := serverBinarySupportsBackupRun(ctx, executor, bin) + changed := false + var policies = map[string]*backup.KindPolicy{} + for i, line := range lines { + m := backup.FindCronMarker(line) + if m == nil { + continue + } + switch m.Form { + case backup.CronFormRun: + // Already the D3 form. + case backup.CronFormCustom: + kept = append(kept, m.Tag) + case backup.CronFormLegacy: + if !supported { + kept = append(kept, m.Tag) + fmt.Fprintf(os.Stderr, "Warning: %s embeds the old bucket/credentials, but %s does not support `backup run` yet — upgrade the server binary and re-run `teploy server backup-target set`\n", m.Tag, bin) + continue + } + // Migrate any retention baked into the legacy line. + if m.KeepLast > 0 || m.MaxAgeDays > 0 { + key := m.App + if m.Accessory != "" { + key = m.App + "/" + m.Accessory + } + policies[key] = &backup.KindPolicy{KeepLast: m.KeepLast, MaxAgeDays: m.MaxAgeDays} + } + lines[i] = m.Cron + " " + buildScheduledBackupRunCmd(bin, m.App, m.Accessory) + " # [" + m.Tag + "]" + rewritten = append(rewritten, m.Tag) + changed = true + } + } + if !changed { + return rewritten, kept, nil + } + if err := executor.RunInput(ctx, "crontab -", strings.NewReader(strings.Join(lines, "\n")+"\n")); err != nil { + return nil, nil, fmt.Errorf("installing the rewritten crontab: %w", err) + } + // Persist migrated retention (best effort — a failure to write the + // policy keeps the lines rewritten; retention just stays as-was). + for key, kp := range policies { + app, acc, _ := strings.Cut(key, "/") + policy, err := backup.ReadPolicy(ctx, executor, app) + if err != nil { + continue + } + if acc == "" { + p := kp + existing := policy.Volumes + if existing != nil && existing.KeepLast == 0 && existing.MaxAgeDays == 0 { + existing.KeepLast, existing.MaxAgeDays = p.KeepLast, p.MaxAgeDays + } else if existing == nil { + policy.Volumes = p + } + } else { + if policy.Accessories == nil { + policy.Accessories = map[string]*backup.KindPolicy{} + } + if policy.Accessories[acc] == nil { + policy.Accessories[acc] = kp + } + } + if err := backup.WritePolicy(ctx, executor, app, policy); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not persist migrated retention for %s: %v\n", key, err) + } + } + return rewritten, kept, nil +} + +// legacyKeepLastRE pulls the keep-last count out of a legacy generated +// volumes line (`... | head -n -N ...`). +var legacyKeepLastRE = regexp.MustCompile(`head -n -([0-9]+)`) + +// recordBackupTargetEvent records backup_target.set for every app known +// on the server: the event log is per app, and the target is shared by +// all of them (detail carries no bucket or credential material). +func recordBackupTargetEvent(ctx context.Context, executor ssh.Executor, err error) { + apps, appsErr := serverAppNames(ctx, executor) + if appsErr != nil { + fmt.Fprintf(os.Stderr, "Warning: could not list apps for the backup_target.set event: %v\n", appsErr) + return + } + for _, app := range apps { + events.Record(ctx, executor, events.Event{App: app, Kind: events.KindBackupTarget, Outcome: events.Outcome(err)}) + } +} + +// serverAppNames lists every app known on the server: .teploy apps with +// history/config plus every deployed state directory. +func serverAppNames(ctx context.Context, executor ssh.Executor) ([]string, error) { + res := ssh.RunDetailed(ctx, executor, `for d in /deployments/.teploy/apps/*; do [ -d "$d" ] && basename -- "$d"; done; for f in /deployments/*/state.json; do [ -e "$f" ] && basename -- "$(dirname -- "$f")"; done`) + if res.Err != nil { + return nil, res.Err + } + if res.ExitCode != 0 { + return nil, fmt.Errorf("%s", res.ExitErrorText()) + } + seen := map[string]bool{} + var apps []string + for _, line := range strings.Split(string(res.Stdout), "\n") { + if app := strings.TrimSpace(line); app != "" && !seen[app] { + seen[app] = true + apps = append(apps, app) + } + } + sort.Strings(apps) + return apps, nil +} diff --git a/internal/cli/backup_target_test.go b/internal/cli/backup_target_test.go new file mode 100644 index 0000000..9353803 --- /dev/null +++ b/internal/cli/backup_target_test.go @@ -0,0 +1,158 @@ +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/ssh" +) + +// Target round-trip: what WriteTarget sends is what ReadTarget parses, and +// the creds flag survives the framing. The mock answers the reader script +// from the payload the writer recorded on stdin. +func TestBackupTargetRoundTrip(t *testing.T) { + mock := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "d=\n", Output: "written /etc/teploy/backup-target.json"}, + ) + ctx := context.Background() + path, fallback, err := backup.WriteTarget(ctx, mock, backup.Target{ + Bucket: "b", Region: "us-east-1", Endpoint: "http://127.0.0.1:9100", Prefix: "prod", + }, "AGE ENCRYPTED") + if err != nil { + t.Fatalf("WriteTarget: %v", err) + } + if path != "/etc/teploy/backup-target.json" || fallback { + t.Fatalf("path=%s fallback=%v", path, fallback) + } + payload := mock.Inputs[len(mock.Inputs)-1] + for _, want := range []string{`"bucket":"b"`, `"region":"us-east-1"`, `"endpoint":"http://127.0.0.1:9100"`, `"prefix":"prod"`, `"credentials":"AGE ENCRYPTED"`} { + if !strings.Contains(payload, want) { + t.Errorf("stored JSON missing %s: %s", want, payload) + } + } + + reader := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + payload}, + ) + target, err := backup.ReadTarget(ctx, reader) + if err != nil { + t.Fatalf("ReadTarget: %v", err) + } + if !TargetConfigured(target) || target.Bucket != "b" || target.Region != "us-east-1" || + target.Endpoint != "http://127.0.0.1:9100" || target.Prefix != "prod" || !target.HasCredentials { + t.Fatalf("round-trip mismatch: %+v", target) + } + if target.Path != "/etc/teploy/backup-target.json" { + t.Fatalf("path not reported: %+v", target) + } +} + +// An absent target reads as unconfigured, never as an error. +func TestBackupTargetAbsent(t *testing.T) { + mock := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent"}, + ) + target, err := backup.ReadTarget(context.Background(), mock) + if err != nil { + t.Fatalf("ReadTarget: %v", err) + } + if TargetConfigured(target) { + t.Fatalf("absent target read as configured: %+v", target) + } +} + +// S3Config.AWS with a stored target decrypts inside the invocation and +// fails closed: the aws command only runs after a non-empty decryption. +func TestS3ConfigAWSStoredTarget(t *testing.T) { + cfg := TargetS3(backup.Target{ + Bucket: "b", Region: "us-east-1", Endpoint: "http://127.0.0.1:9100", + Path: "/etc/teploy/backup-target.json", HasCredentials: true, + }) + cmd := cfg.AWS("s3 ls s3://b/") + for _, want := range []string{ + "python3 -c 'import json,sys; sys.stdout.write(json.load(open(sys.argv[1])).get(\"credentials\") or \"\")' '/etc/teploy/backup-target.json'", + "age -d -i", + `[ -n "$c" ]`, + `eval "$c"`, + "exec aws s3 ls s3://b/ --region 'us-east-1' --endpoint-url 'http://127.0.0.1:9100'", + } { + if !strings.Contains(cmd, want) { + t.Errorf("stored-creds invocation missing %q:\n%s", want, cmd) + } + } + if !strings.HasPrefix(cmd, "( ") || !strings.HasSuffix(cmd, " )") { + t.Errorf("stored-creds invocation must be a self-contained subshell:\n%s", cmd) + } + // No plaintext credential material anywhere. + if strings.Contains(cmd, "AWS_ACCESS_KEY_ID='") || strings.Contains(cmd, "AWS_SECRET_ACCESS_KEY='") { + t.Errorf("stored-creds invocation must not inline credential values:\n%s", cmd) + } +} + +// The prefix reaches every key the CLI builds. +func TestS3ConfigBase(t *testing.T) { + cfg := backup.S3Config{Bucket: "b", Prefix: "prod"} + if cfg.Base() != "b/prod" { + t.Fatalf("Base = %q", cfg.Base()) + } + bare := backup.S3Config{Bucket: "b"} + if bare.Base() != "b" { + t.Fatalf("bare bucket Base changed") + } +} + +// Credential validation refuses anything the run-time eval could misuse. +func TestValidateCredential(t *testing.T) { + for _, ok := range []string{"AKIA1234", "a+b/c=d_e:", "x" + strings.Repeat("y", 200)} { + if err := backup.ValidateCredential("access key", ok); err != nil { + t.Errorf("valid credential rejected: %q: %v", ok, err) + } + } + for _, bad := range []string{"", "with space", "semi;colon", "$(pwned)", "'quoted'", "new\nline", strings.Repeat("x", 256)} { + if err := backup.ValidateCredential("secret key", bad); err == nil { + t.Errorf("dangerous credential accepted: %q", bad) + } + } +} + +func TestValidateEndpointAndPrefix(t *testing.T) { + for _, ok := range []string{"http://127.0.0.1:9100", "https://s3.example.com", "https://s3.example.com:8443/path"} { + if err := backup.ValidateEndpoint(ok); err != nil { + t.Errorf("endpoint %q rejected: %v", ok, err) + } + } + for _, bad := range []string{"ftp://x", "http://host with space", "https://x'quote", "notaurl"} { + if err := backup.ValidateEndpoint(bad); err == nil { + t.Errorf("endpoint %q accepted", bad) + } + } + for _, ok := range []string{"", "prod", "prod/apps", "a.b-c_d"} { + if err := backup.ValidatePrefix(ok); err != nil { + t.Errorf("prefix %q rejected: %v", ok, err) + } + } + for _, bad := range []string{"/lead", "trail/", "a//b", "..", "a/../b"} { + if err := backup.ValidatePrefix(bad); err == nil { + t.Errorf("prefix %q accepted", bad) + } + } +} + +// `backup run` without a stored target fails with the remedy, before any +// docker or aws work. +func TestBackupRunNeedsTarget(t *testing.T) { + mock := ssh.NewMockExecutor("localhost", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent"}, + ) + err := runBackupRunOn(context.Background(), mock, "myapp", "") + if err == nil || !strings.Contains(err.Error(), "no backup target") { + t.Fatalf("err = %v, want the no-target refusal", err) + } + for _, call := range mock.Calls { + if strings.Contains(call, "docker") || strings.Contains(call, "aws s3") { + t.Errorf("no docker/aws work may run without a target: %s", call) + } + } +} diff --git a/internal/cli/contracts_golden_test.go b/internal/cli/contracts_golden_test.go index 090096e..68a6dfb 100644 --- a/internal/cli/contracts_golden_test.go +++ b/internal/cli/contracts_golden_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/config" "github.com/useteploy/teploy/internal/confighome" "github.com/useteploy/teploy/internal/events" @@ -464,6 +465,79 @@ func TestContractsConfigMergeGolden(t *testing.T) { }) } +// TestContractsBackupTargetGolden pins `server backup-target show --json` +// (rev 14, backup-target-v1). Never secret material: only whether +// credentials are stored. +func TestContractsBackupTargetGolden(t *testing.T) { + writeFixture(t, "backup-target-envelope/valid/configured.json", backupTargetDTO{ + MachineInterface: MachineInterface, Host: "192.0.2.10", + Configured: true, + Bucket: "teploy-backups", + Region: "us-east-1", + Endpoint: "http://127.0.0.1:9100", + Prefix: "prod", + HasCredentials: true, + Path: "/etc/teploy/backup-target.json", + }) + writeFixture(t, "backup-target-envelope/valid/unconfigured.json", backupTargetDTO{ + MachineInterface: MachineInterface, Host: "192.0.2.10", + }) +} + +// TestContractsBackupJSONGolden pins the backup-json-v1 envelopes (rev +// 14): `backup list --json` (parsed rows across volumes, accessories and +// teploy data, plus schedules discovered from teploy cron markers), +// `backup create --json` and `backup restore --json` (the quiesced +// restore receipt). +func TestContractsBackupJSONGolden(t *testing.T) { + createdAt := "2026-10-20T03:17:04Z" + writeFixture(t, "backup-list-envelope/valid/app.json", backupListDTO{ + MachineInterface: MachineInterface, Host: "192.0.2.10", App: "blog", + Target: &backupTargetSummary{Bucket: "teploy-backups", Region: "us-east-1", HasCredentials: true}, + Backups: []backup.ListedBackup{ + {ID: "20261020-031704-1f2e3d4c5b6a7988", Kind: "volumes", SizeBytes: 1048576, CreatedAt: createdAt, Key: "s3://teploy-backups/blog/volumes/20261020-031704-1f2e3d4c5b6a7988.tar.gz", Restorable: true}, + {ID: "20261020-031704-1f2e3d4c5b6a7988", Kind: "teploy", SizeBytes: 4096, CreatedAt: createdAt, Key: "s3://teploy-backups/blog/teploy/20261020-031704-1f2e3d4c5b6a7988.tar.gz", Restorable: true}, + {ID: "20261019-030001", Kind: "accessory", Accessory: "db", SizeBytes: 2097152, CreatedAt: "2026-10-19T03:00:01Z", Key: "s3://teploy-backups/blog/accessories/db/20261019-030001.sql.gz", Restorable: true}, + }, + Schedules: []backupScheduleDTO{ + {Kind: "volumes", Cron: "30 3 * * *", KeepLast: 7, Form: "run"}, + {Kind: "accessory", Accessory: "db", Cron: "15 2 * * *", KeepLast: 5, MaxAgeDays: 30, Form: "run"}, + }, + }) + writeFixture(t, "backup-list-envelope/valid/legacy-schedule.json", backupListDTO{ + MachineInterface: MachineInterface, Host: "192.0.2.10", App: "legacy", + Backups: []backup.ListedBackup{}, + Schedules: []backupScheduleDTO{{Kind: "volumes", Cron: "0 3 * * *", KeepLast: 7, Form: "legacy"}}, + }) + writeFixture(t, "backup-create-envelope/valid/volumes.json", backupCreateDTO{ + MachineInterface: MachineInterface, Host: "192.0.2.10", App: "blog", + ID: "20261020-031704-1f2e3d4c5b6a7988", Kind: "volumes", SizeBytes: 1048576, CreatedAt: createdAt, + Key: "s3://teploy-backups/blog/volumes/20261020-031704-1f2e3d4c5b6a7988.tar.gz", + TeployKey: "s3://teploy-backups/blog/teploy/20261020-031704-1f2e3d4c5b6a7988.tar.gz", Pruned: 1, + }) + writeFixture(t, "backup-create-envelope/valid/accessory.json", backupCreateDTO{ + MachineInterface: MachineInterface, Host: "192.0.2.10", App: "blog", + ID: "20261020-031704-1f2e3d4c5b6a7988", Kind: "accessory", Accessory: "db", SizeBytes: 2097152, CreatedAt: createdAt, + Key: "s3://teploy-backups/blog/accessories/db/20261020-031704-1f2e3d4c5b6a7988.sql.gz", + }) + writeFixture(t, "backup-restore-envelope/valid/ok.json", backupRestoreDTO{ + MachineInterface: MachineInterface, Host: "192.0.2.10", App: "blog", + ID: "20261019-030001", Kind: "volumes", OK: true, + Stopped: []string{"blog-web-1f2e3d4", "blog-worker-1f2e3d4"}, + Started: []string{"blog-web-1f2e3d4", "blog-worker-1f2e3d4"}, + Health: "healthy", + }) + writeFixture(t, "backup-restore-envelope/valid/unhealthy.json", backupRestoreDTO{ + MachineInterface: MachineInterface, Host: "192.0.2.10", App: "blog", + ID: "20261019-030001", Kind: "accessory", Accessory: "db", OK: false, + Stopped: []string{"blog-web-1f2e3d4"}, + Started: []string{"blog-web-1f2e3d4"}, + Health: "unhealthy", + RecoveryDir: "/deployments/blog/volumes.restore-old.a1b2c3", + Error: "health check after restore failed: ... — pre-restore data kept in /deployments/blog/volumes.restore-old.a1b2c3", + }) +} + // TestContractsD2Golden pins the D2 folderless-read envelopes (rev 14): // `releases --app --host --json` (container app: records + rollback // availability), `rollback --to X --json` and `redeploy --json` results. @@ -496,6 +570,5 @@ func TestContractsD2Golden(t *testing.T) { }) writeFixture(t, "redeploy-result/valid/folder-home.json", redeployDTO{ MachineInterface: MachineInterface, App: "legacy", Server: "infra-home", Home: "folder", - Release: "abc1234", Image: "example/legacy:abc1234", OK: true, - }) + Release: "abc1234", Image: "example/legacy:abc1234", OK: true}) } diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 7a41925..443f0b2 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/config" "github.com/useteploy/teploy/internal/deploy" "github.com/useteploy/teploy/internal/secret" @@ -202,7 +203,7 @@ func doctorRun(ctx context.Context, deps doctorDeps, flags *Flags, serverName st if executor == nil { const sshDown = "restore SSH connectivity (see the ssh check above), then re-run teploy doctor" - for _, name := range []string{"docker", "disk", "registry", "caddy", "compatibility", "repair-debt", "secret-exposure", "server-tools", "config-home", "age-identity"} { + for _, name := range []string{"docker", "disk", "registry", "caddy", "compatibility", "repair-debt", "secret-exposure", "server-tools", "config-home", "backup-cron", "age-identity"} { skipFail(name, "SSH unreachable", sshDown) } } else { @@ -221,6 +222,7 @@ func doctorRun(ctx context.Context, deps doctorDeps, flags *Flags, serverName st report.Checks = append(report.Checks, doctorSecretExposureCheck(ctx, executor)) report.Checks = append(report.Checks, doctorServerToolsCheck(ctx, executor)) report.Checks = append(report.Checks, doctorConfigHomeCheck(ctx, executor)) + report.Checks = append(report.Checks, doctorBackupCronCheck(ctx, executor)) report.Checks = append(report.Checks, doctorAgeIdentityCheck(ctx, executor)) } @@ -793,6 +795,63 @@ func doctorConfigHomeCheck(ctx context.Context, exec ssh.Executor) doctorCheck { return doctorCheck{Name: "config-home", Result: doctorOK, Detail: "every app is edited in its folder"} } +// doctorBackupCronCheck flags backup cron lines that embed credentials or +// bucket flags (D3). Raw line text is NEVER placed in the report — a +// legacy line can carry plaintext S3 keys, so the check reports markers +// and counts only. +func doctorBackupCronCheck(ctx context.Context, exec ssh.Executor) doctorCheck { + lines, err := backup.ReadCrontab(ctx, exec) + if err != nil { + return doctorCheck{Name: "backup-cron", Result: doctorWarn, Detail: "could not read the crontab: " + err.Error()} + } + var credLines, legacyLines int + var custom, runForm int + for _, line := range lines { + hasCreds := strings.Contains(line, "TEPLOY_S3_SECRET_KEY=") || + strings.Contains(line, "TEPLOY_S3_ACCESS_KEY=") || + strings.Contains(line, "AWS_SECRET_ACCESS_KEY=") || + strings.Contains(line, "AWS_ACCESS_KEY_ID=") + m := backup.FindCronMarker(line) + if hasCreds { + credLines++ + continue + } + if m == nil { + continue + } + switch m.Form { + case backup.CronFormRun: + runForm++ + case backup.CronFormLegacy: + legacyLines++ + case backup.CronFormCustom: + custom++ + } + } + switch { + case credLines > 0: + return doctorCheck{ + Name: "backup-cron", + Result: doctorFail, + Detail: fmt.Sprintf("%d cron line(s) embed S3 credentials in plain text", credLines), + Remediation: "run `teploy server backup-target set` to store the target on the server, which rewrites teploy-managed backup lines to `teploy backup run`; hand-written lines must be fixed by hand", + } + case legacyLines > 0: + return doctorCheck{ + Name: "backup-cron", + Result: doctorWarn, + Detail: fmt.Sprintf("%d teploy-managed backup line(s) still embed the bucket (and possibly credentials)", legacyLines), + Remediation: "run `teploy server backup-target set` to store the target and rewrite them to `teploy backup run`", + } + case custom > 0: + return doctorCheck{Name: "backup-cron", Result: doctorOK, + Detail: fmt.Sprintf("%d scheduled line(s) use `backup run`; %d hand-written line(s) carry a teploy marker and were left alone", runForm, custom)} + default: + return doctorCheck{Name: "backup-cron", Result: doctorOK, + Detail: fmt.Sprintf("%d scheduled line(s) use `backup run`; no embedded credentials or buckets", runForm)} + } +} + // doctorAgeIdentityScript prints three facts about the age identity's // location: the new path (present | denied | absent), the legacy path // (real | marker | denied | absent) and whether this user could move it diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go index 1f96282..ee0c64e 100644 --- a/internal/cli/doctor_test.go +++ b/internal/cli/doctor_test.go @@ -19,8 +19,7 @@ import ( // against (adding checks is additive at the end; renaming/removing is a // machine-interface bump). var doctorCheckOrder = []string{ - "git", "config", "ssh", "docker", "disk", "registry", "caddy", "compatibility", "repair-debt", "secret-exposure", "server-tools", "config-home", "age-identity", -} + "git", "config", "ssh", "docker", "disk", "registry", "caddy", "compatibility", "repair-debt", "secret-exposure", "server-tools", "config-home", "backup-cron", "age-identity"} // doctorHappyMock registers every remote read a fully healthy target // answers, for the app config returned by doctorTestApp. @@ -34,6 +33,7 @@ func doctorHappyMock() *ssh.MockExecutor { ssh.MockCommand{Match: "scan=$(find /deployments ", Output: ""}, ssh.MockCommand{Match: "for t in git python3", Output: ""}, ssh.MockCommand{Match: "for f in /deployments/*/state.json", Output: ""}, + ssh.MockCommand{Match: "crontab -l", Output: ""}, ssh.MockCommand{Match: "if [ -r /etc/teploy/age-key ]", Output: "present labsent movable"}, ) } @@ -83,8 +83,8 @@ func TestDoctorAllChecksPass(t *testing.T) { if ex == nil { t.Fatal("expected an executor back") } - if report.Summary != (doctorSummary{OK: 13, Warn: 0, Fail: 0}) { - t.Fatalf("summary = %+v, want 13 ok", report.Summary) + if report.Summary != (doctorSummary{OK: 14, Warn: 0, Fail: 0}) { + t.Fatalf("summary = %+v, want 14 ok", report.Summary) } if len(report.Checks) != len(doctorCheckOrder) { t.Fatalf("got %d checks, want %d", len(report.Checks), len(doctorCheckOrder)) @@ -131,8 +131,8 @@ func TestDoctorJSONStableShape(t *testing.T) { t.Fatalf("machine_interface = %v, want %d", doc["machine_interface"], MachineInterface) } checks, ok := doc["checks"].([]any) - if !ok || len(checks) != 13 { - t.Fatalf("checks = %#v, want 13 entries", doc["checks"]) + if !ok || len(checks) != 14 { + t.Fatalf("checks = %#v, want 14 entries", doc["checks"]) } for i, raw := range checks { check, ok := raw.(map[string]any) @@ -186,6 +186,7 @@ func TestDoctorHumanTable(t *testing.T) { ssh.MockCommand{Match: "scan=$(find /deployments ", Output: ""}, ssh.MockCommand{Match: "for t in git python3", Output: ""}, ssh.MockCommand{Match: "for f in /deployments/*/state.json", Output: ""}, + ssh.MockCommand{Match: "crontab -l", Output: ""}, ssh.MockCommand{Match: "if [ -r /etc/teploy/age-key ]", Output: "present labsent movable"}, ) report, _ := doctorRun(context.Background(), doctorTestDeps(mock), &Flags{}, "", doctorTestApp(), nil) @@ -200,7 +201,7 @@ func TestDoctorHumanTable(t *testing.T) { if !strings.Contains(rendered, "fix:") { t.Fatalf("table missing the remediation line:\n%s", rendered) } - if !strings.Contains(rendered, "Summary: 12 ok, 0 warn, 1 fail") { + if !strings.Contains(rendered, "Summary: 13 ok, 0 warn, 1 fail") { t.Fatalf("table summary line wrong:\n%s", rendered) } assertDoctorReadOnlyCalls(t, mock.Calls) @@ -673,7 +674,7 @@ func TestDoctorEndToEndAllOK(t *testing.T) { if err := runDoctor(doctorTestDeps(mock), &Flags{}, "", &out); err != nil { t.Fatalf("runDoctor: %v", err) } - if !strings.Contains(out.String(), "Summary: 13 ok, 0 warn, 0 fail") { + if !strings.Contains(out.String(), "Summary: 14 ok, 0 warn, 0 fail") { t.Fatalf("human report wrong:\n%s", out.String()) } assertDoctorReadOnlyCalls(t, mock.Calls) @@ -696,6 +697,7 @@ func assertDoctorReadOnlyCalls(t *testing.T, calls []string) { "find /deployments ", "for t in git python3", "for f in /deployments/*/state.json", + "crontab -l", } for _, call := range calls { allowed := false diff --git a/internal/cli/dr.go b/internal/cli/dr.go index 7359cd9..73ea87a 100644 --- a/internal/cli/dr.go +++ b/internal/cli/dr.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" ) // The disaster-recovery command family (C07). `teploy backup` remains the @@ -44,17 +45,19 @@ func newDRCmd(flags *Flags, version string) *cobra.Command { // drStoreFlags registers the bundle-store selection: an S3 target // (--bucket) or a plain directory reachable on the server (--dir) for -// offline bundles. +// offline bundles. With neither, the server's stored backup target is +// used (D3). func drStoreFlags(cmd *cobra.Command, bucket, region, endpoint, dir *string) { - cmd.Flags().StringVar(bucket, "bucket", "", "S3 bucket for bundles") - cmd.Flags().StringVar(region, "region", "us-east-1", "AWS region") + cmd.Flags().StringVar(bucket, "bucket", "", "S3 bucket for bundles (default: the server's stored backup target)") + cmd.Flags().StringVar(region, "region", "", "AWS region (default: the stored target's region, else us-east-1)") cmd.Flags().StringVar(endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") cmd.Flags().StringVar(dir, "dir", "", "store bundles in a directory on the server (offline bundles) instead of S3") } -// drStore builds the BundleStore from the flags; exactly one target must -// be selected. -func drStore(bucket, region, endpoint, dir string) (backup.BundleStore, error) { +// drStore builds the BundleStore from the flags; with neither --bucket nor +// --dir the server's stored backup target is used (stored credentials and +// all — no bucket or credentials need to travel with the command). +func drStore(ctx context.Context, exec ssh.Executor, bucket, region, endpoint, dir string) (backup.BundleStore, error) { switch { case dir != "" && bucket != "": return nil, fmt.Errorf("choose one bundle target: --bucket (S3) or --dir (server directory), not both") @@ -64,12 +67,23 @@ func drStore(bucket, region, endpoint, dir string) (backup.BundleStore, error) { if err := backup.ValidateBucket(bucket); err != nil { return nil, err } - if err := backup.ValidateRegion(region); err != nil { + r := region + if r == "" { + r = "us-east-1" + } + if err := backup.ValidateRegion(r); err != nil { return nil, err } - return backup.S3BundleStore{S3: s3Config(bucket, region, endpoint)}, nil + return backup.S3BundleStore{S3: s3Config(bucket, r, endpoint)}, nil + } + target, err := backup.ReadTarget(ctx, exec) + if err != nil { + return nil, err } - return nil, fmt.Errorf("a bundle target is required: --bucket (S3) or --dir (server directory)") + if !TargetConfigured(target) { + return nil, fmt.Errorf("a bundle target is required: --bucket (S3), --dir (server directory), or a stored server backup target (`teploy server backup-target set`)") + } + return backup.S3BundleStore{S3: TargetS3(target)}, nil } func newDRCreateCmd(flags *Flags, version string) *cobra.Command { @@ -93,10 +107,6 @@ func newDRCreateCmd(flags *Flags, version string) *cobra.Command { "(--quiesced-volume NAME).", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - store, err := drStore(bucket, region, endpoint, dir) - if err != nil { - return err - } if includeAgeKey && !includeSecrets { return fmt.Errorf("--include-age-key requires --include-secrets (the key decrypts the material it travels with)") } @@ -111,6 +121,10 @@ func newDRCreateCmd(flags *Flags, version string) *cobra.Command { return err } defer executor.Close() + store, err := drStore(ctx, executor, bucket, region, endpoint, dir) + if err != nil { + return err + } progress := io.Writer(os.Stdout) if flags.JSON { progress = os.Stderr @@ -158,10 +172,6 @@ func newDRListCmd(flags *Flags) *cobra.Command { Short: "List DR bundles", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - store, err := drStore(bucket, region, endpoint, dir) - if err != nil { - return err - } appCfg, err := config.LoadApp(".") if err != nil { return err @@ -173,6 +183,10 @@ func newDRListCmd(flags *Flags) *cobra.Command { return err } defer executor.Close() + store, err := drStore(ctx, executor, bucket, region, endpoint, dir) + if err != nil { + return err + } ids, err := store.ListIDs(ctx, executor, appCfg.App) if err != nil { return err @@ -201,10 +215,6 @@ func newDRShowCmd(flags *Flags) *cobra.Command { Short: "Show a DR bundle's manifest", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - store, err := drStore(bucket, region, endpoint, dir) - if err != nil { - return err - } appCfg, err := config.LoadApp(".") if err != nil { return err @@ -216,6 +226,10 @@ func newDRShowCmd(flags *Flags) *cobra.Command { return err } defer executor.Close() + store, err := drStore(ctx, executor, bucket, region, endpoint, dir) + if err != nil { + return err + } data, err := store.FetchManifest(ctx, executor, appCfg.App, args[0]) if err != nil { return err @@ -265,10 +279,6 @@ func newDRRestoreCmd(flags *Flags) *cobra.Command { "BEFORE any mutation.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - store, err := drStore(bucket, region, endpoint, dir) - if err != nil { - return err - } appCfg, err := config.LoadApp(".") if err != nil { return err @@ -280,6 +290,10 @@ func newDRRestoreCmd(flags *Flags) *cobra.Command { return err } defer executor.Close() + store, err := drStore(ctx, executor, bucket, region, endpoint, dir) + if err != nil { + return err + } progress := io.Writer(os.Stdout) if flags.JSON { progress = os.Stderr diff --git a/internal/cli/machineinterface.go b/internal/cli/machineinterface.go index ea1048f..b125b74 100644 --- a/internal/cli/machineinterface.go +++ b/internal/cli/machineinterface.go @@ -142,6 +142,17 @@ const ( // CapUndeployedListV1: `app list --include-undeployed` returns // never-deployed and removed Teploy-home apps in an "undeployed" array. CapUndeployedListV1 = "undeployed-list-v1" + // CapBackupTargetV1: one backup target per server, stored outside + // /deployments (age-encrypted credentials) and read at run time: + // `server backup-target set|show|unset`, backup/DR commands defaulting + // to it, cron lines calling only `backup run`, and `backup-target set` + // rewriting teploy-managed backup cron lines to that form. + CapBackupTargetV1 = "backup-target-v1" + // CapBackupJSONV1: `backup create|list|restore --json` emit parsed, + // versioned envelopes (backup-json-v1 rows: id, kind, accessory, + // size_bytes, created_at, key; schedules discovered from teploy cron + // markers) and restore quiesces the app under its fenced lock. + CapBackupJSONV1 = "backup-json-v1" // CapSecretStdinV1: `secret set KEY --stdin` reads the value verbatim // from stdin (never argv), and secret set/get/list/rm/rotate accept // folderless `--app X --host S` (provider from the app's Teploy @@ -198,12 +209,13 @@ func MachineCapabilities() []string { CapConfigMergeV1, CapConfigUnprivilegedV1, CapDeployHomeV1, + CapBackupTargetV1, + CapBackupJSONV1, CapSecretStdinV1, CapReleasesJsonV1, CapRollbackJsonV1, CapRedeployV1, - CapEnvConfigGuardV1, - } + CapEnvConfigGuardV1} sort.Strings(tokens) return tokens } diff --git a/internal/cli/machineinterface_test.go b/internal/cli/machineinterface_test.go index 064b42b..764e0ad 100644 --- a/internal/cli/machineinterface_test.go +++ b/internal/cli/machineinterface_test.go @@ -104,8 +104,8 @@ func TestCapabilityTokenRegistry(t *testing.T) { "template-var-stdin", "trigger-convergence-v1", } - want = append(want, "actor-flag-v1", "events-v1", "config-home-v1", "undeployed-list-v1", "lock-wait-v1", "config-merge-v1", "config-unprivileged-v1", "deploy-home-v1", - "secret-stdin-v1", "releases-json-v1", "rollback-json-v1", "redeploy-v1", "env-config-guard-v1") + want = append(want, "actor-flag-v1", "events-v1", "config-home-v1", "undeployed-list-v1", "lock-wait-v1", "config-merge-v1", "config-unprivileged-v1", "deploy-home-v1", "backup-target-v1", "backup-json-v1") + want = append(want, "secret-stdin-v1", "releases-json-v1", "rollback-json-v1", "redeploy-v1", "env-config-guard-v1") sort.Strings(want) got := MachineCapabilities() if len(got) != len(want) { diff --git a/internal/cli/server.go b/internal/cli/server.go index 64889df..0ba2e05 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -27,6 +27,7 @@ func newServerCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newServerUpdateCmd(flags)) cmd.AddCommand(newServerListCmd(flags)) cmd.AddCommand(newServerStatusCmd(flags)) + cmd.AddCommand(newServerBackupTargetCmd(flags)) return cmd } diff --git a/internal/deploy/lifecycle.go b/internal/deploy/lifecycle.go index fd08e1e..3798873 100644 --- a/internal/deploy/lifecycle.go +++ b/internal/deploy/lifecycle.go @@ -198,6 +198,124 @@ func (l *Lifecycle) Restart(ctx context.Context, app string, timeout int) error return nil } +// QuiesceApp stops the app's running PROCESS containers (accessories +// excluded — an accessory restore stops its own container only when its +// data is restored as files) under the caller's fenced lock, returning +// the stopped containers in stop order. An app with no deploy state has +// nothing to stop and returns nil (restoring an undeployed app's volumes +// is legal). Used by the quiesced backup restore (D3). +func (l *Lifecycle) QuiesceApp(ctx context.Context, lk *state.Lock, app string, timeout int) ([]docker.Container, error) { + current, err := state.Read(ctx, l.exec, app) + if err != nil { + return nil, err + } + if current == nil { + return nil, nil + } + containers, err := l.appContainersByVersion(ctx, app, current.CurrentHash) + if err != nil { + return nil, err + } + var out []docker.Container + for _, c := range containers { + if c.State == "running" && c.Labels["teploy.role"] != "accessory" { + out = append(out, c) + } + } + for _, c := range out { + fmt.Fprintf(l.out, "Stopping %s...\n", c.Name) + if err := l.stop(ctx, lk, c, timeout); err != nil { + return out, err + } + } + return out, nil +} + +// ResumeReport says what ResumeApp did. +type ResumeReport struct { + Started []string + HealthErr error +} + +// HealthStatus renders the health outcome for machine consumers. +func (r ResumeReport) HealthStatus() string { + switch { + case r.HealthErr != nil: + return "unhealthy" + case len(r.Started) == 0: + return "skipped" + default: + return "healthy" + } +} + +// ResumeApp starts containers previously stopped by QuiesceApp (or any +// guarded stop under the same lock) and health-checks the app's web +// process exactly as Start does. nil containers means nothing was stopped +// (undeployed app): the health check is skipped. A start or health +// failure is returned in HealthErr — the caller decides what to keep +// (a failed restore keeps its recovery copy). +func (l *Lifecycle) ResumeApp(ctx context.Context, lk *state.Lock, app string, containers []docker.Container) ResumeReport { + report := ResumeReport{} + for _, c := range containers { + fmt.Fprintf(l.out, "Starting %s...\n", c.Name) + if err := l.start(ctx, lk, c); err != nil { + report.HealthErr = err + return report + } + report.Started = append(report.Started, c.Name) + } + if len(containers) == 0 { + return report + } + if err := l.healthCheckCurrent(ctx, app); err != nil { + report.HealthErr = err + } + return report +} + +// healthCheckCurrent probes the current release's web container on its +// recorded port — the same check Start and Restart run after starting. +func (l *Lifecycle) healthCheckCurrent(ctx context.Context, app string) error { + current, err := state.Read(ctx, l.exec, app) + if err != nil || current == nil { + return fmt.Errorf("no deploy state found for %s", app) + } + if current.CurrentPort <= 0 { + return nil + } + fmt.Fprintln(l.out, "Running health check...") + deployer := &Deployer{exec: l.exec, out: l.out} + containers, err := l.appContainersByVersion(ctx, app, current.CurrentHash) + if err != nil { + return err + } + webName := "" + for _, c := range containers { + if c.Labels["teploy.process"] == "web" { + webName = c.Name + break + } + } + if webName == "" { + return fmt.Errorf("current release has no web container") + } + bindHost := l.docker.HostBindIP(ctx, webName) + health := defaultHealthConfig() + rec, recErr := releasemeta.Read(ctx, l.exec, app, current.CurrentHash) + if recErr != nil { + return recErr + } + if rec != nil { + applyRecordToRollback(&RollbackConfig{}, rec, &health) + } + if err := deployer.healthCheck(ctx, current.CurrentPort, health, bindHost); err != nil { + return fmt.Errorf("health check failed after start: %w", err) + } + fmt.Fprintln(l.out, " Health check passed") + return nil +} + // Only current recorded processes and currently configured accessories belong // to lifecycle. Start activates configured accessories, including previously // stopped ones; removed/unknown accessories and historical workloads stay alone. diff --git a/internal/events/events.go b/internal/events/events.go index cd8d3e2..a5ae1b3 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -109,6 +109,9 @@ const ( KindMaintenanceOff = "maintenance.off" KindGateRepair = "gate.repair" KindWatermarkReset = "watermark.reset" + KindBackupRun = "backup.run" + KindBackupRestore = "backup.restore" + KindBackupTarget = "backup_target.set" ) // detailAllow is the per-kind allowlist of detail keys. Env and secret @@ -139,6 +142,9 @@ var detailAllow = map[string][]string{ KindMaintenanceOff: {"error"}, KindGateRepair: {"case", "message"}, KindWatermarkReset: {"ref"}, + KindBackupRun: {"id", "kind", "accessory", "error"}, + KindBackupRestore: {"id", "kind", "accessory", "error"}, + KindBackupTarget: {"id", "kind", "accessory", "error"}, } // maxErrorDetail bounds the "error" detail so a command's full stderr