From 05d2b28bf863c924d501aca45c2866794e6da78f Mon Sep 17 00:00:00 2001 From: lex00 <121451605+lex00@users.noreply.github.com> Date: Thu, 16 Jul 2026 10:36:30 -0600 Subject: [PATCH] feat: model a machine that fails to reach `started` (image-pull failure injection) (#61) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lifecycle advancer only ever drove a machine to `started`; `StateFailed` was defined but never set, so a Machines-API client's deploy-failure path couldn't be tested. A machine whose `config.image` is the sentinel `mudflaps/unpullable` (bare, or with a `:tag`/`@digest`) now settles into `failed` instead of `started` on its next boot transition — modeling a boot-time image-pull failure ("unable to pull image, canceling deploy"). The boot transition (`settle`) reads the machine's latest config when it fires, so an update to a real image beforehand still boots. A client that polls `.../wait?state=started` sees a deploy that never comes up: the wait times out (`408`, never a false `started`) and `GET` shows `state: failed`. Everything stays deterministic on the injected clock. Real flaps has no such magic image; the sentinel is the single injection point. Tests cover the advancer failed-boot path, sentinel matching (tag/digest, nil config, non-matches), update-to-good-image recovery, and the server-level deploy-failure path (create → failed → wait 408). README + fidelity docs note the injection. Staged as 0.5.0. --- CHANGELOG.md | 20 ++++++- README.md | 1 + docs/fidelity.md | 26 ++++++++- internal/flaps/types.go | 29 ++++++++++ internal/machine/machine.go | 44 ++++++++++++--- internal/machine/machine_test.go | 91 ++++++++++++++++++++++++++++++++ internal/server/server_test.go | 54 +++++++++++++++++++ 7 files changed, 255 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c87b7..7b6954e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.5.0] - 2026-07-16 + +### Added + +- Deploy-failure injection: a machine whose `config.image` is the sentinel + `mudflaps/unpullable` (bare, or with a `:tag`/`@digest`) settles into `failed` + instead of `started` on its next boot transition (create/start/restart/update), + modeling a boot-time image-pull failure ("unable to pull image, not found, + canceling deploy"). A client that waits for `started` sees a deploy that never + comes up — `.../wait?state=started` times out (`408`, never a false `started`) + and `GET .../machines/{id}` reports `state: failed`. The decision is read from + the machine's latest config when the boot transition fires, so an update back + to a real image before then still boots. This is the single injection point + that lets a Machines-API client (e.g. an IaC applier) exercise its + deploy-failure/cancel path offline and deterministically — real flaps has no + such magic image (#61). + ## [0.4.1] - 2026-07-14 ### Fixed @@ -122,7 +139,8 @@ Fidelity and correctness pass from an adversarial audit against `superfly/fly-go - Distroless container image, GoReleaser configuration, mkdocs-material doc site, and CI. -[Unreleased]: https://github.com/intentius/mudflaps/compare/v0.4.1...HEAD +[Unreleased]: https://github.com/intentius/mudflaps/compare/v0.5.0...HEAD +[0.5.0]: https://github.com/intentius/mudflaps/compare/v0.4.1...v0.5.0 [0.4.1]: https://github.com/intentius/mudflaps/compare/v0.4.0...v0.4.1 [0.4.0]: https://github.com/intentius/mudflaps/compare/v0.3.1...v0.4.0 [0.3.1]: https://github.com/intentius/mudflaps/compare/v0.3.0...v0.3.1 diff --git a/README.md b/README.md index 2bd89c0..86bcb66 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Testing a Machines-API client, such as an infrastructure-as-code applier, requir - Suspend/resume, cordon/uncordon, and machine metadata endpoints. - Machine leases with nonce, TTL, owner, and expiry; a mutation by a non-holder returns `409`, and a conflicting acquire returns the holder's lease envelope (without the nonce). - Synchronous version churn: an update mints a new `instance_id`, marks the prior version `replaced`, and returns the new version in the response. +- Deploy-failure injection: a machine whose `config.image` is the sentinel `mudflaps/unpullable` settles into `failed` instead of `started`, modeling a boot-time image-pull failure ("unable to pull image, canceling deploy"). A client that waits for `started` observes a deploy that never comes up — its failure path, tested offline. - `/wait` long-poll that blocks until a target state or returns `408`; the deadline is measured on the injected clock, so timeouts are deterministic in tests. - A `/_mudflaps/health` endpoint that reports the implemented paths; the roadmap is currently empty — every documented endpoint mudflaps targets is built. - Single static binary and distroless container image; no runtime dependencies. diff --git a/docs/fidelity.md b/docs/fidelity.md index 79911b7..ab29c0f 100644 --- a/docs/fidelity.md +++ b/docs/fidelity.md @@ -16,6 +16,7 @@ stateDiagram-v2 [*] --> created: POST machines (skip_launch) creating --> starting starting --> started + starting --> failed: unpullable image started --> stopping: POST stop stopping --> stopped stopped --> starting: POST start @@ -33,7 +34,8 @@ stateDiagram-v2 The states the emulator actually enters fall into three groups: -- Resting: `created`, `started`, `stopped`, `suspended`. +- Resting: `created`, `started`, `stopped`, `suspended`, and `failed` (see + [Deploy-failure injection](#deploy-failure-injection)). - Transient: `creating`, `starting`, `stopping`, `restarting`, `suspending`, `replacing`, `destroying`. - Terminal: `destroyed` (a destroyed machine is reaped from the store), and @@ -43,6 +45,28 @@ A `skip_launch` create rests directly at `created` rather than booting. A destroyed machine is removed, so a subsequent operation returns `404` rather than resurrecting it. +## Deploy-failure injection + +Real deploys don't always succeed: a machine flaps accepts can still fail to +pull its image at boot ("unable to pull image, not found, canceling deploy") and +never reach `started`. mudflaps models the Machines-API state layer, not an image +registry, so that failure can't arise on its own — but a client's deploy-failure +handling still needs testing. + +A machine whose `config.image` is the sentinel **`mudflaps/unpullable`** (bare, +or with a `:tag`/`@digest`) settles into `failed` instead of `started` on its +next boot transition — create, start, restart, or an update to that image. The +decision is read from the machine's latest config at the moment the boot +transition fires, so an update back to a real image before then still boots +normally. + +For a client that polls `.../wait?state=started`, this is a deploy that never +comes up: the wait returns `408` (it never falsely reports `started`), and a +`GET` of the machine shows `state: failed`. Everything stays deterministic on the +injected clock. Real flaps has no such magic image; the sentinel is the single +injection point that lets an applier exercise its failure/cancel/rollback path +offline. + ## Leases and nonces A lease grants its holder exclusive rights to mutate a machine. The holder diff --git a/internal/flaps/types.go b/internal/flaps/types.go index 9adf377..b6afdda 100644 --- a/internal/flaps/types.go +++ b/internal/flaps/types.go @@ -12,6 +12,8 @@ // so on) must match the real wire format exactly. package flaps +import "strings" + // MachineState is the lifecycle state of a machine. Fly models three groups of // states: persistent resting states, transient in-flight states, and terminal // states a machine never leaves. @@ -26,6 +28,33 @@ const ( StateFailed MachineState = "failed" ) +// UnpullableImage is a sentinel image reference. A machine whose config.image is +// this value (with or without a `:tag` or `@digest`) never reaches `started`: +// its boot transition settles into StateFailed instead, modeling a machine that +// flaps accepted but that could not pull its image at boot ("Unable to pull +// image, not found, canceling deploy" — a common real-world deploy failure). +// +// Real flaps has no such magic value; mudflaps models the Machines-API *state* +// layer, not an image registry, so a boot-time pull failure can't arise on its +// own. This sentinel is the injection point that lets a Machines-API client +// (e.g. an IaC applier that polls .../wait for `started`) exercise its +// deploy-failure path offline and deterministically. See issue #61. +const UnpullableImage = "mudflaps/unpullable" + +// FailsToBoot reports whether a machine with this config should settle into +// StateFailed rather than StateStarted on its next boot transition — i.e. its +// image is the UnpullableImage sentinel (bare, or with a `:tag`/`@digest`). +// Safe on a nil receiver. +func (c *MachineConfig) FailsToBoot() bool { + if c == nil { + return false + } + img := c.Image + return img == UnpullableImage || + strings.HasPrefix(img, UnpullableImage+":") || + strings.HasPrefix(img, UnpullableImage+"@") +} + // Transient states: a machine passes through these while an operation runs. const ( StateCreating MachineState = "creating" diff --git a/internal/machine/machine.go b/internal/machine/machine.go index 862945e..f0bd736 100644 --- a/internal/machine/machine.go +++ b/internal/machine/machine.go @@ -57,21 +57,24 @@ func NewAdvancer(s *store.Store, clk clock.Clock, delays Delays, log *slog.Logge return &Advancer{store: s, clk: clk, delays: delays, log: log} } -// Create moves a freshly created machine creating -> starting -> started. +// Create moves a freshly created machine creating -> starting -> started (or, +// for a machine whose image can't be pulled, creating -> starting -> failed; +// see settle). func (a *Advancer) Create(app, machineID string) { a.clk.AfterFunc(a.delays.Create, func() { a.set(app, machineID, flaps.StateStarting) a.clk.AfterFunc(a.delays.Start, func() { - a.set(app, machineID, flaps.StateStarted) + a.settle(app, machineID) }) }) } -// Start moves a stopped machine starting -> started. The transient state is set -// by the caller (the server) before scheduling. +// Start moves a stopped machine starting -> started (or -> failed for an +// unpullable image). The transient state is set by the caller (the server) +// before scheduling. func (a *Advancer) Start(app, machineID string) { a.clk.AfterFunc(a.delays.Start, func() { - a.set(app, machineID, flaps.StateStarted) + a.settle(app, machineID) }) } @@ -82,10 +85,11 @@ func (a *Advancer) Stop(app, machineID string) { }) } -// Restart moves a machine restarting -> started. +// Restart moves a machine restarting -> started (or -> failed for an unpullable +// image). func (a *Advancer) Restart(app, machineID string) { a.clk.AfterFunc(a.delays.Restart, func() { - a.set(app, machineID, flaps.StateStarted) + a.settle(app, machineID) }) } @@ -114,10 +118,34 @@ func (a *Advancer) Destroy(app, machineID string) { // new version replacing -> started. func (a *Advancer) Update(app, machineID string) { a.clk.AfterFunc(a.delays.Update, func() { - a.set(app, machineID, flaps.StateStarted) + a.settle(app, machineID) }) } +// settle drives a booting machine to its resting state: StateStarted normally, +// or StateFailed when its config marks it unpullable (see +// flaps.MachineConfig.FailsToBoot). This is the one place a machine can end in +// `failed` — modeling a boot-time image-pull failure so a client that waits for +// `started` observes a deploy that never comes up (issue #61). Reading the +// config inside the same mutation keeps the decision consistent with the very +// latest update (an update to a good image before this fires still boots). +func (a *Advancer) settle(app, machineID string) { + _, err := a.store.UpdateMachine(app, machineID, func(m *flaps.Machine) error { + state := flaps.StateStarted + if m.Config.FailsToBoot() { + state = flaps.StateFailed + } + m.State = state + if n := len(m.Versions); n > 0 { + m.Versions[n-1].State = state + } + return nil + }) + if err != nil { + a.log.Debug("settle skipped", "app", app, "machine", machineID, "err", err) + } +} + // set writes a resting or terminal state onto a machine, keeping the current // version entry's state in step. func (a *Advancer) set(app, machineID string, state flaps.MachineState) { diff --git a/internal/machine/machine_test.go b/internal/machine/machine_test.go index e5e6f91..9a58eee 100644 --- a/internal/machine/machine_test.go +++ b/internal/machine/machine_test.go @@ -92,6 +92,97 @@ func TestCreateReachesStartedInOneAdvance(t *testing.T) { } } +// seedImage seeds a creating machine whose config carries the given image, so +// the boot transition can consult it (see settle / FailsToBoot). +func seedImage(t *testing.T, s *store.Store, image string) { + t.Helper() + m := flaps.Machine{ + ID: "m1", + State: flaps.StateCreating, + InstanceID: "INSTANCE0", + Config: &flaps.MachineConfig{Image: image}, + Versions: []flaps.MachineVersion{{InstanceID: "INSTANCE0", State: flaps.StateCreating}}, + } + if _, err := s.CreateMachine("demo", m); err != nil { + t.Fatalf("CreateMachine: %v", err) + } +} + +// #61 — a machine whose image can't be pulled settles into `failed`, not +// `started`: the boot transition consults the config sentinel. +func TestCreateReachesFailedForUnpullableImage(t *testing.T) { + s, clk, a := newFixture(t) + seedImage(t, s, flaps.UnpullableImage) + + a.Create("demo", "m1") + + // creating -> starting (the boot is still pending). + clk.Advance(time.Second) + if got := stateOf(t, s); got != flaps.StateStarting { + t.Fatalf("after first step state = %q, want starting", got) + } + + // starting -> failed (not started). + clk.Advance(time.Second) + if got := stateOf(t, s); got != flaps.StateFailed { + t.Fatalf("after boot state = %q, want failed", got) + } + + // The version history tracks the terminal state too. + m, err := s.GetMachine("demo", "m1") + if err != nil { + t.Fatalf("GetMachine: %v", err) + } + if n := len(m.Versions); n == 0 || m.Versions[n-1].State != flaps.StateFailed { + t.Fatalf("version state = %v, want failed", m.Versions) + } +} + +func TestUnpullableImageMatchesTagAndDigest(t *testing.T) { + for _, img := range []string{ + flaps.UnpullableImage, + flaps.UnpullableImage + ":latest", + flaps.UnpullableImage + "@sha256:deadbeef", + } { + if !(&flaps.MachineConfig{Image: img}).FailsToBoot() { + t.Fatalf("FailsToBoot(%q) = false, want true", img) + } + } + for _, img := range []string{"nginx:1", flaps.UnpullableImage + "-not", "", "mudflaps/unpullableX"} { + if (&flaps.MachineConfig{Image: img}).FailsToBoot() { + t.Fatalf("FailsToBoot(%q) = true, want false", img) + } + } + // nil config never fails to boot. + var c *flaps.MachineConfig + if c.FailsToBoot() { + t.Fatal("nil config FailsToBoot() = true, want false") + } +} + +// An update to a good image before the boot fires still reaches `started`: +// settle reads the latest config, so a recovering update wins. +func TestUpdateToGoodImageRecoversFromUnpullable(t *testing.T) { + s, clk, a := newFixture(t) + seedImage(t, s, flaps.UnpullableImage) + + a.Create("demo", "m1") + clk.Advance(time.Second) // creating -> starting + + // Client fixes the image before the boot transition fires. + if _, err := s.UpdateMachine("demo", "m1", func(m *flaps.Machine) error { + m.Config.Image = "nginx:1" + return nil + }); err != nil { + t.Fatalf("UpdateMachine: %v", err) + } + + clk.Advance(time.Second) // starting -> started (image is good now) + if got := stateOf(t, s); got != flaps.StateStarted { + t.Fatalf("state = %q, want started", got) + } +} + func TestStopAndDestroy(t *testing.T) { s, clk, a := newFixture(t) seed(t, s) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 7f29aa4..cd0e34b 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -137,6 +137,60 @@ func TestCreateAndWaitStarted(t *testing.T) { } } +// TestUnpullableImageDeployFails is the #61 deploy-failure path: a machine +// created with the UnpullableImage sentinel settles into `failed` (modeling a +// boot-time image-pull failure), so a client that waits for `started` sees a +// deploy that never comes up — mirroring the forum report this feature targets. +func TestUnpullableImageDeployFails(t *testing.T) { + h := newHarness(t) + if code, body := h.do(http.MethodPost, "/v1/apps", flaps.CreateAppRequest{AppName: "demo", OrgSlug: "personal"}, nil); code != http.StatusCreated { + t.Fatalf("create app: %d %s", code, body) + } + code, body := h.do(http.MethodPost, "/v1/apps/demo/machines", flaps.CreateMachineRequest{ + Config: &flaps.MachineConfig{Image: flaps.UnpullableImage}, + }, nil) + if code != http.StatusOK { + t.Fatalf("create machine: %d %s", code, body) + } + var m flaps.Machine + h.mustJSON(body, &m) + + h.clk.Advance(time.Hour) // fire creating -> starting -> failed + + // The machine is observable as failed (how a client learns the deploy died). + code, body = h.do(http.MethodGet, "/v1/apps/demo/machines/"+m.ID, nil, nil) + if code != http.StatusOK { + t.Fatalf("get machine = %d %s", code, body) + } + var got flaps.Machine + h.mustJSON(body, &got) + if got.State != flaps.StateFailed { + t.Fatalf("machine state = %q, want failed", got.State) + } + + // A wait for `started` never falsely reports success — it times out (408). + done := make(chan int, 1) + go func() { + c, _ := h.do(http.MethodGet, "/v1/apps/demo/machines/"+m.ID+"/wait?state=started&timeout=1", nil, nil) + done <- c + }() + safety := time.After(3 * time.Second) + for { + select { + case c := <-done: + if c != http.StatusRequestTimeout { + t.Fatalf("wait-for-started on a failed machine = %d, want 408", c) + } + return + case <-safety: + t.Fatal("wait-for-started on a failed machine did not time out") + default: + h.clk.Advance(500 * time.Millisecond) + time.Sleep(2 * time.Millisecond) + } + } +} + // TestCreateAppRequiresOrgSlug mirrors real Fly, which rejects app creation // without an org_slug (400) rather than silently creating the app. func TestCreateAppRequiresOrgSlug(t *testing.T) {