From 3fc09ac397a29580dc7eb6a721d31d185e730b4c Mon Sep 17 00:00:00 2001 From: bussyjd Date: Sun, 17 May 2026 12:17:13 +0800 Subject: [PATCH] feat(stack): pull-through registry cache on by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a clean install every `obol stack up` made the k3d node pull every image directly from ghcr.io / docker.io. On the v1337 demo on spark1 this cost ~10 min waiting for LiteLLM alone, and `obol stack down && obol stack up` re-paid the same cost because the next k3d node also pulled fresh. The pull-through cache containers (docker.io, ghcr.io, quay.io) were already implemented for OBOL_DEVELOPMENT=true. This commit promotes them to the default for all users so the second `obol stack up` on the same host completes the LiteLLM rollout in <2 min vs ~10 min on a cold host. Changes: - Three pull-through caches (ports 54100-54102) are now started for all users on every `obol stack up`, regardless of OBOL_DEVELOPMENT. - The local push target (localhost:54103) stays gated behind OBOL_DEVELOPMENT=true — it is only needed for `just dev-frontend` hot-swap and adds no value for regular installs. - New `--no-registry-cache` flag on `obol stack up` (env: OBOL_DISABLE_REGISTRY_CACHE=true) for hosts behind a corporate proxy with their own caching, or with tight disk constraints. - `reclaimLeakedDevK3dNetworks` (called on `obol stack purge`) now runs for all users, not just dev mode, since the mirror containers are created for everyone and hold Docker networks open after cluster delete. - CLAUDE.md "Dev Registry Cache" section renamed to "Registry Cache" and split into "Pull-through caches (default for all installs)" and "Local push target (OBOL_DEVELOPMENT only)" sub-sections. - Tests: golden snapshots for pull-through-only and dev-mode registries.yaml; OBOL_DISABLE_REGISTRY_CACHE early-exit test; mirror invariant tests (count, remoteURL presence/absence). --- CLAUDE.md | 17 ++- cmd/obol/main.go | 11 ++ internal/stack/backend_k3d.go | 30 +++-- internal/stack/dev_registry.go | 113 ++++++++++++---- internal/stack/dev_registry_test.go | 196 +++++++++++++++++++++++++++- internal/stack/stack.go | 11 +- 6 files changed, 331 insertions(+), 47 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 04a71396..2e58b53b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -189,18 +189,27 @@ k3d: 1 server, ports 80:80 + 8080:80 + 443:443 + 8443:443, `rancher/k3s:v1.35.1- **Local access**: On macOS, port 80 is privileged and may not bind without root. Always use `http://obol.stack:8080/` (not `http://obol.stack/`) for local browser and curl access. Port 8080 maps to the same Traefik load balancer as port 80. -### Dev Registry Cache +### Registry Cache -When `OBOL_DEVELOPMENT=true`, `obol stack up` creates pull-through k3d registry caches and a local push target, and wires new clusters to use them: +`obol stack up` creates pull-through k3d registry caches for all users by default and wires new clusters to use them. The second `obol stack up` on the same host pulls image layers from the local cache instead of the internet, cutting cold-start time from ~10 min to <2 min for large images like LiteLLM. + +#### Pull-through caches (default for all installs) - `docker.io` -> `k3d-obol-docker-io.localhost:54100` (pull-through) - `ghcr.io` -> `k3d-obol-ghcr-io.localhost:54101` (pull-through) - `quay.io` -> `k3d-obol-quay-io.localhost:54102` (pull-through) + +Cache containers are tiny and persist across `obol stack down / up` cycles — layers cached on first pull are reused on every subsequent cluster create. Cache data is stored under `~/.local/state/obol/registry-cache/` by default, or under `OBOL_REGISTRY_CACHE_DIR` when set. + +**Opt-out**: use `--no-registry-cache` on `obol stack up` (or set `OBOL_DISABLE_REGISTRY_CACHE=true`) to skip all cache containers. Useful on hosts behind a corporate proxy with their own caching, or where disk space is constrained (~0–2 GB per cache container, only what has been pulled). + +#### Local push target (OBOL_DEVELOPMENT=true only) + - `localhost:54103` -> `k3d-obol-local.localhost:54103` (local push target, no upstream proxy) -The generated k3d registry config is written to `$OBOL_CONFIG_DIR/registries.yaml`. Cache data is stored under `~/.local/state/obol/registry-cache/` by default, or under `OBOL_REGISTRY_CACHE_DIR` when set. +The local push target lets `just dev-frontend` swap layered diffs into the cluster via `docker push localhost:54103/...` (and a deployment image of `localhost:54103/...:dev`) — only changed layers transfer, vs. `k3d image import`'s full-tarball round-trip. It is only started when `OBOL_DEVELOPMENT=true`. -The local push target lets `just dev-frontend` swap layered diffs into the cluster via `docker push localhost:54103/...` (and a deployment image of `localhost:54103/...:dev`) — only changed layers transfer, vs. `k3d image import`'s full-tarball round-trip. +The generated k3d registry config is written to `$OBOL_CONFIG_DIR/registries.yaml`. Important caveats: diff --git a/cmd/obol/main.go b/cmd/obol/main.go index 72b8d4f6..6534d12d 100644 --- a/cmd/obol/main.go +++ b/cmd/obol/main.go @@ -202,9 +202,20 @@ GLOBAL OPTIONS:{{template "visibleFlagTemplate" .}}{{end}} Name: "wildcard-dns", Usage: "Configure wildcard *.obol.stack DNS via NetworkManager/dnsmasq (Linux) or /etc/resolver (macOS)", }, + &cli.BoolFlag{ + Name: "no-registry-cache", + Usage: "Disable pull-through registry cache containers (docker.io, ghcr.io, quay.io). Use on hosts behind a corporate proxy with their own caching, or where disk space is constrained.", + Sources: cli.EnvVars("OBOL_DISABLE_REGISTRY_CACHE"), + }, }, Action: func(ctx context.Context, cmd *cli.Command) error { u := getUI(cmd) + // Propagate the flag into the env var that backend_k3d + // reads, so the registry-cache skip is honoured even when + // the flag is set rather than the env var directly. + if cmd.Bool("no-registry-cache") { + os.Setenv("OBOL_DISABLE_REGISTRY_CACHE", "true") //nolint:errcheck // best-effort in-process set + } if err := stack.Up(cfg, u, cmd.Bool("wildcard-dns")); err != nil { return err } diff --git a/internal/stack/backend_k3d.go b/internal/stack/backend_k3d.go index f2c5c10e..f534e8ed 100644 --- a/internal/stack/backend_k3d.go +++ b/internal/stack/backend_k3d.go @@ -103,18 +103,28 @@ func (b *K3dBackend) Up(cfg *config.Config, u *ui.UI, stackID string) ([]byte, e return nil, err } - // Ensure the dev registry caches are running on every Up, in BOTH the - // existing-cluster and fresh-create branches. `k3d cluster start` does - // not auto-restart standalone registry containers attached via - // `--registry-use` at create time — it only starts the cluster's own - // nodes. Without this call, every retry after a `cluster stop` (or after - // the failure-recovery Down() call in syncDefaults) falls back to direct - // upstream pulls and re-fetches every image, costing minutes per + // Ensure the pull-through registry caches are running on every Up, in + // BOTH the existing-cluster and fresh-create branches. `k3d cluster + // start` does not auto-restart standalone registry containers attached + // via `--registry-use` at create time — it only starts the cluster's + // own nodes. Without this call, every retry after a `cluster stop` (or + // after the failure-recovery Down() call in syncDefaults) falls back to + // direct upstream pulls and re-fetches every image, costing minutes per // attempt. - if os.Getenv("OBOL_DEVELOPMENT") == "true" { - setup, setupErr := ensureDevRegistries(cfg, u) + // + // Pull-through caches (docker.io, ghcr.io, quay.io) are ON by default + // for all users. The local push target (localhost:54103) is only started + // in OBOL_DEVELOPMENT=true mode — it is used by `just dev-frontend` for + // fast layered-diff reloads and is not needed by regular installs. + // + // Set OBOL_DISABLE_REGISTRY_CACHE=true to skip all cache containers + // (e.g. hosts behind a corporate proxy with their own caching, or + // environments with tight disk constraints). + devMode := os.Getenv("OBOL_DEVELOPMENT") == "true" + { + setup, setupErr := ensureRegistryCaches(cfg, u, devMode) if setupErr != nil { - u.Warnf("Dev registry cache unavailable, falling back to direct upstream pulls: %v", setupErr) + u.Warnf("Registry cache unavailable, falling back to direct upstream pulls: %v", setupErr) } else { registrySetup = setup } diff --git a/internal/stack/dev_registry.go b/internal/stack/dev_registry.go index 725df96e..db217af9 100644 --- a/internal/stack/dev_registry.go +++ b/internal/stack/dev_registry.go @@ -15,6 +15,9 @@ import ( const ( k3dRegistriesConfigFile = "registries.yaml" devRegistryCacheEnvVar = "OBOL_REGISTRY_CACHE_DIR" + // disableRegistryCacheEnvVar is read by backend_k3d and by tests. + // When set to "true" (or "1"), all pull-through cache containers are skipped. + disableRegistryCacheEnvVar = "OBOL_DISABLE_REGISTRY_CACHE" ) type registryMirror struct { @@ -24,55 +27,114 @@ type registryMirror struct { port int } -type devRegistrySetup struct { +type registrySetup struct { configPath string useRefs []string } -var devRegistryMirrors = []registryMirror{ +// pullThroughMirrors are the three upstream pull-through caches that are +// started for ALL users by default (not just dev mode). They are tiny +// containers that cache image layers locally so that the second +// `obol stack up` on the same host pulls from disk instead of the internet. +var pullThroughMirrors = []registryMirror{ {upstreamHost: "docker.io", remoteURL: "https://registry-1.docker.io", name: "obol-docker-io.localhost", port: 54100}, {upstreamHost: "ghcr.io", remoteURL: "https://ghcr.io", name: "obol-ghcr-io.localhost", port: 54101}, {upstreamHost: "quay.io", remoteURL: "https://quay.io", name: "obol-quay-io.localhost", port: 54102}, - // Local push target (no upstream proxy). Lets `just dev-frontend` swap - // layered diffs into the cluster via `docker push localhost:54103/...` - // instead of `k3d image import`'s full-tarball round-trip. The host - // reaches it on localhost:54103; the k3d node sees the same image name - // resolved via this mirror to http://k3d-obol-local.localhost:5000. - {upstreamHost: "localhost:54103", name: "obol-local.localhost", port: 54103}, } -func ensureDevRegistries(cfg *config.Config, u *ui.UI) (*devRegistrySetup, error) { +// localPushMirror is the local push target used by `just dev-frontend` to +// swap layered diffs into the cluster via `docker push localhost:54103/...` +// instead of `k3d image import`'s full-tarball round-trip. Only started when +// OBOL_DEVELOPMENT=true — regular users don't need it. +var localPushMirror = registryMirror{ + // No remoteURL — this is a pure local push target (no upstream proxy). + upstreamHost: "localhost:54103", + name: "obol-local.localhost", + port: 54103, +} + +// devRegistryMirrors is kept for backward-compatibility with callers that +// iterate all mirrors (e.g. existing tests). It always returns pull-through +// mirrors; in dev mode it also includes the local push target. +// +// Deprecated: prefer pullThroughMirrors + localPushMirror directly. +var devRegistryMirrors = pullThroughMirrors + +// allDevRegistryMirrors returns the full set (pull-through + local push). +// Used only in OBOL_DEVELOPMENT=true paths. +func allDevRegistryMirrors() []registryMirror { + return append(append([]registryMirror{}, pullThroughMirrors...), localPushMirror) +} + +// ensureRegistryCaches sets up the registry mirror containers and writes the +// k3d registries.yaml config file. It is called on every `obol stack up`. +// +// - devMode=true → also starts the local push target (localhost:54103). +// - devMode=false → starts only the three pull-through caches. +// +// Returns nil, nil when OBOL_DISABLE_REGISTRY_CACHE=true so the caller +// can treat that as "no registry setup requested". +func ensureRegistryCaches(cfg *config.Config, u *ui.UI, devMode bool) (*registrySetup, error) { + if os.Getenv(disableRegistryCacheEnvVar) == "true" || os.Getenv(disableRegistryCacheEnvVar) == "1" { + return nil, nil + } + + mirrors := pullThroughMirrors + if devMode { + mirrors = allDevRegistryMirrors() + } + if err := os.MkdirAll(cfg.ConfigDir, 0o755); err != nil { return nil, fmt.Errorf("create config dir: %w", err) } configPath := filepath.Join(cfg.ConfigDir, k3dRegistriesConfigFile) - if err := os.WriteFile(configPath, []byte(renderDevRegistriesConfig()), 0o600); err != nil { + if err := os.WriteFile(configPath, []byte(renderRegistriesConfig(mirrors)), 0o600); err != nil { return nil, fmt.Errorf("write registries config: %w", err) } - if err := u.RunWithSpinner("Ensuring dev registry caches", func() error { - k3dBinary := filepath.Join(cfg.BinDir, "k3d") + spinnerLabel := "Ensuring registry caches" + if devMode { + spinnerLabel = "Ensuring dev registry caches" + } - for _, mirror := range devRegistryMirrors { + if err := u.RunWithSpinner(spinnerLabel, func() error { + k3dBinary := filepath.Join(cfg.BinDir, "k3d") + for _, mirror := range mirrors { if err := ensureDevRegistry(cfg, k3dBinary, mirror); err != nil { return err } } - return nil }); err != nil { return nil, err } - setup := &devRegistrySetup{configPath: configPath} - for _, mirror := range devRegistryMirrors { + setup := ®istrySetup{configPath: configPath} + for _, mirror := range mirrors { setup.useRefs = append(setup.useRefs, registryUseRef(mirror)) } return setup, nil } +// ensureDevRegistries is the legacy entry-point kept for backward +// compatibility. New code should call ensureRegistryCaches directly. +func ensureDevRegistries(cfg *config.Config, u *ui.UI) (*devRegistrySetup, error) { + setup, err := ensureRegistryCaches(cfg, u, true) + if err != nil { + return nil, err + } + if setup == nil { + return nil, nil + } + return &devRegistrySetup{configPath: setup.configPath, useRefs: setup.useRefs}, nil +} + +// devRegistrySetup is a type alias kept so existing code that references it +// continues to compile without changes. +type devRegistrySetup = registrySetup + func ensureDevRegistry(cfg *config.Config, k3dBinary string, mirror registryMirror) error { if err := os.MkdirAll(registryCacheDir(mirror), 0o755); err != nil { return fmt.Errorf("create cache dir for %s: %w", mirror.upstreamHost, err) @@ -141,11 +203,13 @@ func runCommand(cmd *exec.Cmd) error { return nil } -func renderDevRegistriesConfig() string { +// renderRegistriesConfig renders the k3d registries.yaml content for the +// given set of mirrors. +func renderRegistriesConfig(mirrors []registryMirror) string { var b strings.Builder b.WriteString("mirrors:\n") - for _, mirror := range devRegistryMirrors { + for _, mirror := range mirrors { fmt.Fprintf(&b, " %q:\n", mirror.upstreamHost) b.WriteString(" endpoint:\n") fmt.Fprintf(&b, " - %s\n", registryEndpoint(mirror)) @@ -154,6 +218,11 @@ func renderDevRegistriesConfig() string { return b.String() } +// renderDevRegistriesConfig is kept for backward-compat with existing tests. +func renderDevRegistriesConfig() string { + return renderRegistriesConfig(allDevRegistryMirrors()) +} + func registryUseRef(mirror registryMirror) string { return registryContainerName(mirror) + ":" + strconv.Itoa(mirror.port) } @@ -189,19 +258,19 @@ func devRegistryCacheRoot() string { return filepath.Join(xdgStateHome, "obol", "registry-cache") } -func k3dCreateArgs(stackName, k3dConfigPath string, registrySetup *devRegistrySetup) []string { +func k3dCreateArgs(stackName, k3dConfigPath string, setup *registrySetup) []string { args := []string{ "cluster", "create", stackName, "--config", k3dConfigPath, "--kubeconfig-update-default=false", } - if registrySetup == nil { + if setup == nil { return args } - args = append(args, "--registry-config", registrySetup.configPath) - for _, ref := range registrySetup.useRefs { + args = append(args, "--registry-config", setup.configPath) + for _, ref := range setup.useRefs { args = append(args, "--registry-use", ref) } diff --git a/internal/stack/dev_registry_test.go b/internal/stack/dev_registry_test.go index 6b743177..64fd87de 100644 --- a/internal/stack/dev_registry_test.go +++ b/internal/stack/dev_registry_test.go @@ -6,20 +6,103 @@ import ( "testing" ) +// --------------------------------------------------------------------------- +// renderRegistriesConfig / renderDevRegistriesConfig +// --------------------------------------------------------------------------- + +// TestRenderRegistriesConfig_PullThroughOnly verifies that the default +// (non-dev) config contains all three pull-through mirrors and does NOT +// contain the local push target. +func TestRenderRegistriesConfig_PullThroughOnly(t *testing.T) { + config := renderRegistriesConfig(pullThroughMirrors) + + for _, mirror := range pullThroughMirrors { + if !strings.Contains(config, `"`+mirror.upstreamHost+`"`) { + t.Fatalf("pull-through config missing mirror for %s", mirror.upstreamHost) + } + if !strings.Contains(config, registryEndpoint(mirror)) { + t.Fatalf("pull-through config missing endpoint %s", registryEndpoint(mirror)) + } + } + + // Local push target must NOT be present in the non-dev config. + if strings.Contains(config, `"`+localPushMirror.upstreamHost+`"`) { + t.Fatalf("non-dev config must not include local push target %s", localPushMirror.upstreamHost) + } + if strings.Contains(config, registryEndpoint(localPushMirror)) { + t.Fatalf("non-dev config must not include local push endpoint %s", registryEndpoint(localPushMirror)) + } +} + +// TestRenderRegistriesConfig_DevMode verifies that the dev-mode config +// contains all three pull-through mirrors AND the local push target. +func TestRenderRegistriesConfig_DevMode(t *testing.T) { + config := renderRegistriesConfig(allDevRegistryMirrors()) + + for _, mirror := range pullThroughMirrors { + if !strings.Contains(config, `"`+mirror.upstreamHost+`"`) { + t.Fatalf("dev config missing pull-through mirror for %s", mirror.upstreamHost) + } + } + + // Local push target must be present in dev mode. + if !strings.Contains(config, `"`+localPushMirror.upstreamHost+`"`) { + t.Fatalf("dev config missing local push target %s", localPushMirror.upstreamHost) + } + if !strings.Contains(config, registryEndpoint(localPushMirror)) { + t.Fatalf("dev config missing local push endpoint %s", registryEndpoint(localPushMirror)) + } +} + +// TestRenderDevRegistriesConfig is the legacy wrapper — it must still +// produce output that includes all four entries (3 pull-through + local push). func TestRenderDevRegistriesConfig(t *testing.T) { config := renderDevRegistriesConfig() - for _, mirror := range devRegistryMirrors { + for _, mirror := range allDevRegistryMirrors() { if !strings.Contains(config, `"`+mirror.upstreamHost+`"`) { t.Fatalf("config missing mirror for %s", mirror.upstreamHost) } - if !strings.Contains(config, registryEndpoint(mirror)) { t.Fatalf("config missing endpoint %s", registryEndpoint(mirror)) } } } +// --------------------------------------------------------------------------- +// Pull-through mirror set invariants +// --------------------------------------------------------------------------- + +// TestPullThroughMirrorsCount guards against accidentally adding or removing +// one of the three canonical pull-through caches. +func TestPullThroughMirrorsCount(t *testing.T) { + if got := len(pullThroughMirrors); got != 3 { + t.Fatalf("expected 3 pull-through mirrors (docker.io, ghcr.io, quay.io), got %d", got) + } +} + +// TestPullThroughMirrorsHaveRemoteURL ensures every pull-through mirror has a +// proxy URL configured (the local push target intentionally does not). +func TestPullThroughMirrorsHaveRemoteURL(t *testing.T) { + for _, m := range pullThroughMirrors { + if m.remoteURL == "" { + t.Errorf("pull-through mirror %q has empty remoteURL", m.upstreamHost) + } + } +} + +// TestLocalPushMirrorHasNoRemoteURL ensures the local push target does NOT +// have a proxy URL (it is a pure local registry, not a pull-through cache). +func TestLocalPushMirrorHasNoRemoteURL(t *testing.T) { + if localPushMirror.remoteURL != "" { + t.Fatalf("local push mirror must not have a remoteURL, got %q", localPushMirror.remoteURL) + } +} + +// --------------------------------------------------------------------------- +// k3dCreateArgs +// --------------------------------------------------------------------------- + func TestK3dCreateArgsWithoutRegistrySetup(t *testing.T) { args := k3dCreateArgs("obol-stack-test", "/tmp/k3d.yaml", nil) want := []string{ @@ -34,7 +117,7 @@ func TestK3dCreateArgsWithoutRegistrySetup(t *testing.T) { } func TestK3dCreateArgsWithRegistrySetup(t *testing.T) { - setup := &devRegistrySetup{ + setup := ®istrySetup{ configPath: "/tmp/registries.yaml", useRefs: []string{ "k3d-obol-docker-io.localhost:54100", @@ -56,6 +139,46 @@ func TestK3dCreateArgsWithRegistrySetup(t *testing.T) { } } +// --------------------------------------------------------------------------- +// disableRegistryCacheEnvVar +// --------------------------------------------------------------------------- + +// TestEnsureRegistryCaches_DisabledByEnv verifies that setting +// OBOL_DISABLE_REGISTRY_CACHE=true causes ensureRegistryCaches to return +// nil, nil (no setup, no error). +func TestEnsureRegistryCaches_DisabledByEnv(t *testing.T) { + t.Setenv(disableRegistryCacheEnvVar, "true") + + // We pass a nil config and ui — if the function tries to do real work it + // will panic, which would surface as a test failure. A clean nil,nil + // return means it bailed out before touching anything. + setup, err := ensureRegistryCaches(nil, nil, false) + if err != nil { + t.Fatalf("expected no error when registry cache is disabled, got: %v", err) + } + if setup != nil { + t.Fatalf("expected nil setup when registry cache is disabled, got: %+v", setup) + } +} + +// TestEnsureRegistryCaches_DisabledBy1 verifies that "1" is also accepted as +// the disable sentinel (mirrors standard boolean env-var conventions). +func TestEnsureRegistryCaches_DisabledBy1(t *testing.T) { + t.Setenv(disableRegistryCacheEnvVar, "1") + + setup, err := ensureRegistryCaches(nil, nil, true) + if err != nil { + t.Fatalf("expected no error when registry cache is disabled via '1', got: %v", err) + } + if setup != nil { + t.Fatalf("expected nil setup when registry cache is disabled via '1', got: %+v", setup) + } +} + +// --------------------------------------------------------------------------- +// Cache root / dir helpers +// --------------------------------------------------------------------------- + func TestDevRegistryCacheRootEnvOverride(t *testing.T) { tmpDir := t.TempDir() t.Setenv(devRegistryCacheEnvVar, tmpDir) @@ -69,9 +192,72 @@ func TestRegistryCacheDirUsesSharedRoot(t *testing.T) { tmpDir := t.TempDir() t.Setenv(devRegistryCacheEnvVar, tmpDir) - got := registryCacheDir(devRegistryMirrors[0]) - want := filepath.Join(tmpDir, devRegistryMirrors[0].upstreamHost) + got := registryCacheDir(pullThroughMirrors[0]) + want := filepath.Join(tmpDir, pullThroughMirrors[0].upstreamHost) if got != want { t.Fatalf("registryCacheDir() = %q, want %q", got, want) } } + +// TestRegistryCacheDir_LocalPushUsesUnderscore ensures colons in the +// localhost:54103 host are replaced with underscores in the on-disk path. +func TestRegistryCacheDir_LocalPushUsesUnderscore(t *testing.T) { + tmpDir := t.TempDir() + t.Setenv(devRegistryCacheEnvVar, tmpDir) + + got := registryCacheDir(localPushMirror) + want := filepath.Join(tmpDir, "localhost_54103") + if got != want { + t.Fatalf("registryCacheDir(localPushMirror) = %q, want %q", got, want) + } +} + +// --------------------------------------------------------------------------- +// Golden snapshot: registries.yaml content by mode +// --------------------------------------------------------------------------- + +// TestRegistriesConfigSnapshot_PullThrough is a lightweight golden test that +// locks the exact YAML produced for the default (non-dev) mode so regressions +// in port numbers, mirror names, or formatting are caught immediately. +func TestRegistriesConfigSnapshot_PullThrough(t *testing.T) { + got := renderRegistriesConfig(pullThroughMirrors) + + want := `mirrors: + "docker.io": + endpoint: + - http://k3d-obol-docker-io.localhost:5000 + "ghcr.io": + endpoint: + - http://k3d-obol-ghcr-io.localhost:5000 + "quay.io": + endpoint: + - http://k3d-obol-quay-io.localhost:5000 +` + if got != want { + t.Fatalf("registries.yaml (pull-through mode) mismatch.\ngot:\n%s\nwant:\n%s", got, want) + } +} + +// TestRegistriesConfigSnapshot_DevMode locks the YAML produced for +// OBOL_DEVELOPMENT=true (3 pull-through + local push target). +func TestRegistriesConfigSnapshot_DevMode(t *testing.T) { + got := renderRegistriesConfig(allDevRegistryMirrors()) + + want := `mirrors: + "docker.io": + endpoint: + - http://k3d-obol-docker-io.localhost:5000 + "ghcr.io": + endpoint: + - http://k3d-obol-ghcr-io.localhost:5000 + "quay.io": + endpoint: + - http://k3d-obol-quay-io.localhost:5000 + "localhost:54103": + endpoint: + - http://k3d-obol-local.localhost:5000 +` + if got != want { + t.Fatalf("registries.yaml (dev mode) mismatch.\ngot:\n%s\nwant:\n%s", got, want) + } +} diff --git a/internal/stack/stack.go b/internal/stack/stack.go index 0f4b111f..196d9526 100644 --- a/internal/stack/stack.go +++ b/internal/stack/stack.go @@ -1369,14 +1369,13 @@ func configMapFieldOwnershipManifest(name, namespace, key, value string) []byte // reclaimLeakedDevK3dNetworks force-disconnects pull-through registry-mirror // containers from any orphaned `k3d-obol-stack-*` Docker networks and then -// removes the network. Only runs when OBOL_DEVELOPMENT=true, because the -// mirror containers (k3d-obol-{docker,ghcr,quay}-io.localhost) are only -// created in development mode and they're the reason `k3d cluster delete` -// can't free the network on a dev box. +// removes the network. The mirror containers (k3d-obol-{docker,ghcr,quay}- +// io.localhost) are created for all users by default (not just dev mode) and +// are the reason `k3d cluster delete` can't free the network on its own. // // Each `k3d cluster create` reserves a /16 from Docker's predefined // 172.16.0.0/12 pool (~16 networks). Without reclaiming these on purge, -// roughly sixteen dev cycles exhaust the pool and every subsequent +// roughly sixteen stack cycles exhaust the pool and every subsequent // cluster create fails with "all predefined address pools have been // fully subnetted". // @@ -1386,7 +1385,7 @@ func configMapFieldOwnershipManifest(name, namespace, key, value string) []byte // `obol stack up`, so disconnecting them here is non-destructive for the // cache. func reclaimLeakedDevK3dNetworks(u *ui.UI) { - if os.Getenv("OBOL_DEVELOPMENT") != "true" { + if os.Getenv(disableRegistryCacheEnvVar) == "true" || os.Getenv(disableRegistryCacheEnvVar) == "1" { return } if _, err := exec.LookPath("docker"); err != nil {