feat(stack): pull-through registry cache on by default

CLAUDE.md

@@ -189,18 +189,27 @@ k3d: 1 server, ports 80:80 + 8080:80 + 443:443 + 8443:443, `rancher/k3s:v1.35.1-
 
 **Local access**: On macOS, port 80 is privileged and may not bind without root. Always use `http://obol.stack:8080/` (not `http://obol.stack/`) for local browser and curl access. Port 8080 maps to the same Traefik load balancer as port 80.
 
-### Dev Registry Cache
+### Registry Cache
 
-When `OBOL_DEVELOPMENT=true`, `obol stack up` creates pull-through k3d registry caches and a local push target, and wires new clusters to use them:
+`obol stack up` creates pull-through k3d registry caches for all users by default and wires new clusters to use them. The second `obol stack up` on the same host pulls image layers from the local cache instead of the internet, cutting cold-start time from ~10 min to <2 min for large images like LiteLLM.
+
+#### Pull-through caches (default for all installs)
 
 - `docker.io` -> `k3d-obol-docker-io.localhost:54100` (pull-through)
 - `ghcr.io` -> `k3d-obol-ghcr-io.localhost:54101` (pull-through)
 - `quay.io` -> `k3d-obol-quay-io.localhost:54102` (pull-through)
+
+Cache containers are tiny and persist across `obol stack down / up` cycles — layers cached on first pull are reused on every subsequent cluster create. Cache data is stored under `~/.local/state/obol/registry-cache/` by default, or under `OBOL_REGISTRY_CACHE_DIR` when set.
+
+**Opt-out**: use `--no-registry-cache` on `obol stack up` (or set `OBOL_DISABLE_REGISTRY_CACHE=true`) to skip all cache containers. Useful on hosts behind a corporate proxy with their own caching, or where disk space is constrained (~0–2 GB per cache container, only what has been pulled).
+
+#### Local push target (OBOL_DEVELOPMENT=true only)
+
 - `localhost:54103` -> `k3d-obol-local.localhost:54103` (local push target, no upstream proxy)
 
-The generated k3d registry config is written to `$OBOL_CONFIG_DIR/registries.yaml`. Cache data is stored under `~/.local/state/obol/registry-cache/` by default, or under `OBOL_REGISTRY_CACHE_DIR` when set.
+The local push target lets `just dev-frontend` swap layered diffs into the cluster via `docker push localhost:54103/...` (and a deployment image of `localhost:54103/...:dev`) — only changed layers transfer, vs. `k3d image import`'s full-tarball round-trip. It is only started when `OBOL_DEVELOPMENT=true`.
 
-The local push target lets `just dev-frontend` swap layered diffs into the cluster via `docker push localhost:54103/...` (and a deployment image of `localhost:54103/...:dev`) — only changed layers transfer, vs. `k3d image import`'s full-tarball round-trip.
+The generated k3d registry config is written to `$OBOL_CONFIG_DIR/registries.yaml`.
 
 Important caveats:
 

cmd/obol/main.go

@@ -202,9 +202,20 @@ GLOBAL OPTIONS:{{template "visibleFlagTemplate" .}}{{end}}
 								Name:  "wildcard-dns",
 								Usage: "Configure wildcard *.obol.stack DNS via NetworkManager/dnsmasq (Linux) or /etc/resolver (macOS)",
 							},
+							&cli.BoolFlag{
+								Name:    "no-registry-cache",
+								Usage:   "Disable pull-through registry cache containers (docker.io, ghcr.io, quay.io). Use on hosts behind a corporate proxy with their own caching, or where disk space is constrained.",
+								Sources: cli.EnvVars("OBOL_DISABLE_REGISTRY_CACHE"),
+							},
 						},
 						Action: func(ctx context.Context, cmd *cli.Command) error {
 							u := getUI(cmd)
+							// Propagate the flag into the env var that backend_k3d
+							// reads, so the registry-cache skip is honoured even when
+							// the flag is set rather than the env var directly.
+							if cmd.Bool("no-registry-cache") {
+								os.Setenv("OBOL_DISABLE_REGISTRY_CACHE", "true") //nolint:errcheck // best-effort in-process set
+							}
 							if err := stack.Up(cfg, u, cmd.Bool("wildcard-dns")); err != nil {
 								return err
 							}

internal/stack/backend_k3d.go

@@ -103,18 +103,28 @@ func (b *K3dBackend) Up(cfg *config.Config, u *ui.UI, stackID string) ([]byte, e
 		return nil, err
 	}
 
-	// Ensure the dev registry caches are running on every Up, in BOTH the
-	// existing-cluster and fresh-create branches. `k3d cluster start` does
-	// not auto-restart standalone registry containers attached via
-	// `--registry-use` at create time — it only starts the cluster's own
-	// nodes. Without this call, every retry after a `cluster stop` (or after
-	// the failure-recovery Down() call in syncDefaults) falls back to direct
-	// upstream pulls and re-fetches every image, costing minutes per
+	// Ensure the pull-through registry caches are running on every Up, in
+	// BOTH the existing-cluster and fresh-create branches. `k3d cluster
+	// start` does not auto-restart standalone registry containers attached
+	// via `--registry-use` at create time — it only starts the cluster's
+	// own nodes. Without this call, every retry after a `cluster stop` (or
+	// after the failure-recovery Down() call in syncDefaults) falls back to
+	// direct upstream pulls and re-fetches every image, costing minutes per
 	// attempt.
-	if os.Getenv("OBOL_DEVELOPMENT") == "true" {
-		setup, setupErr := ensureDevRegistries(cfg, u)
+	//
+	// Pull-through caches (docker.io, ghcr.io, quay.io) are ON by default
+	// for all users. The local push target (localhost:54103) is only started
+	// in OBOL_DEVELOPMENT=true mode — it is used by `just dev-frontend` for
+	// fast layered-diff reloads and is not needed by regular installs.
+	//
+	// Set OBOL_DISABLE_REGISTRY_CACHE=true to skip all cache containers
+	// (e.g. hosts behind a corporate proxy with their own caching, or
+	// environments with tight disk constraints).
+	devMode := os.Getenv("OBOL_DEVELOPMENT") == "true"
+	{
+		setup, setupErr := ensureRegistryCaches(cfg, u, devMode)
 		if setupErr != nil {
-			u.Warnf("Dev registry cache unavailable, falling back to direct upstream pulls: %v", setupErr)
+			u.Warnf("Registry cache unavailable, falling back to direct upstream pulls: %v", setupErr)
 		} else {
 			registrySetup = setup
 		}

internal/stack/dev_registry.go

@@ -15,6 +15,9 @@ import (
 const (
 	k3dRegistriesConfigFile = "registries.yaml"
 	devRegistryCacheEnvVar  = "OBOL_REGISTRY_CACHE_DIR"
+	// disableRegistryCacheEnvVar is read by backend_k3d and by tests.
+	// When set to "true" (or "1"), all pull-through cache containers are skipped.
+	disableRegistryCacheEnvVar = "OBOL_DISABLE_REGISTRY_CACHE"
 )
 
 type registryMirror struct {
@@ -24,55 +27,114 @@ type registryMirror struct {
 	port         int
 }
 
-type devRegistrySetup struct {
+type registrySetup struct {
 	configPath string
 	useRefs    []string
 }
 
-var devRegistryMirrors = []registryMirror{
+// pullThroughMirrors are the three upstream pull-through caches that are
+// started for ALL users by default (not just dev mode). They are tiny
+// containers that cache image layers locally so that the second
+// `obol stack up` on the same host pulls from disk instead of the internet.
+var pullThroughMirrors = []registryMirror{
 	{upstreamHost: "docker.io", remoteURL: "https://registry-1.docker.io", name: "obol-docker-io.localhost", port: 54100},
 	{upstreamHost: "ghcr.io", remoteURL: "https://ghcr.io", name: "obol-ghcr-io.localhost", port: 54101},
 	{upstreamHost: "quay.io", remoteURL: "https://quay.io", name: "obol-quay-io.localhost", port: 54102},
-	// Local push target (no upstream proxy). Lets `just dev-frontend` swap
-	// layered diffs into the cluster via `docker push localhost:54103/...`
-	// instead of `k3d image import`'s full-tarball round-trip. The host
-	// reaches it on localhost:54103; the k3d node sees the same image name
-	// resolved via this mirror to http://k3d-obol-local.localhost:5000.
-	{upstreamHost: "localhost:54103", name: "obol-local.localhost", port: 54103},
 }
 
-func ensureDevRegistries(cfg *config.Config, u *ui.UI) (*devRegistrySetup, error) {
+// localPushMirror is the local push target used by `just dev-frontend` to
+// swap layered diffs into the cluster via `docker push localhost:54103/...`
+// instead of `k3d image import`'s full-tarball round-trip. Only started when
+// OBOL_DEVELOPMENT=true — regular users don't need it.
+var localPushMirror = registryMirror{
+	// No remoteURL — this is a pure local push target (no upstream proxy).
+	upstreamHost: "localhost:54103",
+	name:         "obol-local.localhost",
+	port:         54103,
+}
+
+// devRegistryMirrors is kept for backward-compatibility with callers that
+// iterate all mirrors (e.g. existing tests). It always returns pull-through
+// mirrors; in dev mode it also includes the local push target.
+//
+// Deprecated: prefer pullThroughMirrors + localPushMirror directly.
+var devRegistryMirrors = pullThroughMirrors
+
+// allDevRegistryMirrors returns the full set (pull-through + local push).
+// Used only in OBOL_DEVELOPMENT=true paths.
+func allDevRegistryMirrors() []registryMirror {
+	return append(append([]registryMirror{}, pullThroughMirrors...), localPushMirror)
+}
+
+// ensureRegistryCaches sets up the registry mirror containers and writes the
+// k3d registries.yaml config file. It is called on every `obol stack up`.
+//
+//   - devMode=true → also starts the local push target (localhost:54103).
+//   - devMode=false → starts only the three pull-through caches.
+//
+// Returns nil, nil when OBOL_DISABLE_REGISTRY_CACHE=true so the caller
+// can treat that as "no registry setup requested".
+func ensureRegistryCaches(cfg *config.Config, u *ui.UI, devMode bool) (*registrySetup, error) {
+	if os.Getenv(disableRegistryCacheEnvVar) == "true" || os.Getenv(disableRegistryCacheEnvVar) == "1" {
+		return nil, nil
+	}
+
+	mirrors := pullThroughMirrors
+	if devMode {
+		mirrors = allDevRegistryMirrors()
+	}
+
 	if err := os.MkdirAll(cfg.ConfigDir, 0o755); err != nil {
 		return nil, fmt.Errorf("create config dir: %w", err)
 	}
 
 	configPath := filepath.Join(cfg.ConfigDir, k3dRegistriesConfigFile)
-	if err := os.WriteFile(configPath, []byte(renderDevRegistriesConfig()), 0o600); err != nil {
+	if err := os.WriteFile(configPath, []byte(renderRegistriesConfig(mirrors)), 0o600); err != nil {
 		return nil, fmt.Errorf("write registries config: %w", err)
 	}
 
-	if err := u.RunWithSpinner("Ensuring dev registry caches", func() error {
-		k3dBinary := filepath.Join(cfg.BinDir, "k3d")
+	spinnerLabel := "Ensuring registry caches"
+	if devMode {
+		spinnerLabel = "Ensuring dev registry caches"
+	}
 
-		for _, mirror := range devRegistryMirrors {
+	if err := u.RunWithSpinner(spinnerLabel, func() error {
+		k3dBinary := filepath.Join(cfg.BinDir, "k3d")
+		for _, mirror := range mirrors {
 			if err := ensureDevRegistry(cfg, k3dBinary, mirror); err != nil {
 				return err
 			}
 		}
-
 		return nil
 	}); err != nil {
 		return nil, err
 	}
 
-	setup := &devRegistrySetup{configPath: configPath}
-	for _, mirror := range devRegistryMirrors {
+	setup := &registrySetup{configPath: configPath}
+	for _, mirror := range mirrors {
 		setup.useRefs = append(setup.useRefs, registryUseRef(mirror))
 	}
 
 	return setup, nil
 }
 
+// ensureDevRegistries is the legacy entry-point kept for backward
+// compatibility. New code should call ensureRegistryCaches directly.
+func ensureDevRegistries(cfg *config.Config, u *ui.UI) (*devRegistrySetup, error) {
+	setup, err := ensureRegistryCaches(cfg, u, true)
+	if err != nil {
+		return nil, err
+	}
+	if setup == nil {
+		return nil, nil
+	}
+	return &devRegistrySetup{configPath: setup.configPath, useRefs: setup.useRefs}, nil
+}
+
+// devRegistrySetup is a type alias kept so existing code that references it
+// continues to compile without changes.
+type devRegistrySetup = registrySetup
+
 func ensureDevRegistry(cfg *config.Config, k3dBinary string, mirror registryMirror) error {
 	if err := os.MkdirAll(registryCacheDir(mirror), 0o755); err != nil {
 		return fmt.Errorf("create cache dir for %s: %w", mirror.upstreamHost, err)
@@ -141,11 +203,13 @@ func runCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
-func renderDevRegistriesConfig() string {
+// renderRegistriesConfig renders the k3d registries.yaml content for the
+// given set of mirrors.
+func renderRegistriesConfig(mirrors []registryMirror) string {
 	var b strings.Builder
 
 	b.WriteString("mirrors:\n")
-	for _, mirror := range devRegistryMirrors {
+	for _, mirror := range mirrors {
 		fmt.Fprintf(&b, "  %q:\n", mirror.upstreamHost)
 		b.WriteString("    endpoint:\n")
 		fmt.Fprintf(&b, "      - %s\n", registryEndpoint(mirror))
@@ -154,6 +218,11 @@ func renderDevRegistriesConfig() string {
 	return b.String()
 }
 
+// renderDevRegistriesConfig is kept for backward-compat with existing tests.
+func renderDevRegistriesConfig() string {
+	return renderRegistriesConfig(allDevRegistryMirrors())
+}
+
 func registryUseRef(mirror registryMirror) string {
 	return registryContainerName(mirror) + ":" + strconv.Itoa(mirror.port)
 }
@@ -189,19 +258,19 @@ func devRegistryCacheRoot() string {
 	return filepath.Join(xdgStateHome, "obol", "registry-cache")
 }
 
-func k3dCreateArgs(stackName, k3dConfigPath string, registrySetup *devRegistrySetup) []string {
+func k3dCreateArgs(stackName, k3dConfigPath string, setup *registrySetup) []string {
 	args := []string{
 		"cluster", "create", stackName,
 		"--config", k3dConfigPath,
 		"--kubeconfig-update-default=false",
 	}
 
-	if registrySetup == nil {
+	if setup == nil {
 		return args
 	}
 
-	args = append(args, "--registry-config", registrySetup.configPath)
-	for _, ref := range registrySetup.useRefs {
+	args = append(args, "--registry-config", setup.configPath)
+	for _, ref := range setup.useRefs {
 		args = append(args, "--registry-use", ref)
 	}