From 438f7c94ff9ee8219e81dd65b4cb60b0d6a2ba4a Mon Sep 17 00:00:00 2001 From: Valentin Volkl Date: Wed, 1 Jul 2026 14:41:45 +0200 Subject: [PATCH 1/2] lease: target dedicated gateway graft endpoint for DirectGraft Adapt the gateway client from the body-flag graft API (cvmfs PR #4268, "direct_graft":true in the commit body) to the dedicated graft endpoint (cvmfs PR #4296, POST /api/v1/leases//graft). PR #4268 was never merged; PR #4296 is the shape that landed, and it selects the DirectGraft fast path by request type rather than a wire flag. The commit body is identical for both endpoints, so DirectGraft now only changes the target URL: Commit and CommitFinalizeOnly route to .../graft when set, and the now-unused "direct_graft" field is dropped from the body. The URL/body construction that was duplicated across both methods moves into gatewayCommitURL/gatewayCommitBody helpers. The --gateway-direct-graft flag and the DirectGraft field are unchanged; only the underlying wire mechanism moved. REFERENCE.md documents the new endpoint and notes it requires the graft support from cvmfs PR #4296 (the standard lease/payload/commit endpoints still need no gateway changes). Assisted-by: Claude (claude-opus-4-8) --- REFERENCE.md | 25 +++++++++-- internal/api/orchestrator.go | 10 +++-- internal/lease/backend.go | 11 ++--- internal/lease/lease.go | 81 +++++++++++++++++++++++------------- internal/lease/lease_test.go | 78 ++++++++++++++++++++++++++++++++++ 5 files changed, 162 insertions(+), 43 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index 1a510f1..7393d85 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -335,12 +335,29 @@ prepub service itself defines: |---|---| | `POST /api/v1/leases/{path}` | Reserve a path-scoped lease for a sub-path | | `POST /api/v1/payloads` | Submit pre-processed catalog diff + objects | -| `DELETE /api/v1/leases/{token}` | Release the lease (commit or abort) | +| `POST /api/v1/leases/{token}` | Commit the lease (standard `DiffRec` merge) | +| `POST /api/v1/leases/{token}/graft` | Commit the lease via the DirectGraft fast path (see below) | +| `DELETE /api/v1/leases/{token}` | Release the lease (abort) | > The real gateway does not implement `PUT /api/v1/leases/{token}` (it returns > `405`); there is no client-side lease heartbeat against this endpoint. -The pre-publisher targets this API directly as a first-class gateway client. No gateway modifications are required. +**DirectGraft fast path.** When the publisher is publishing a brand-new +directory subtree — nothing to diff against the parent — it can request the +`.../graft` endpoint instead of the standard commit. The gateway then grafts the +pre-built subtree catalog directly into the parent catalog +(`WritableCatalogManager::TryGraftNestedCatalog`), skipping the `DiffRec` +catalog merge entirely. The request body is **identical** to a standard commit +(`old_root_hash`, `new_root_hash`, `tag_name`, `tag_description`); the dedicated +endpoint — not a body flag — selects the fast path. It is toggled client-side by +`--gateway-direct-graft` (the `DirectGraft` commit field) and is a pure +performance optimisation: both paths produce identical repository state for the +"publish new subtree" case. + +The pre-publisher targets this API directly as a first-class gateway client. The +standard commit, payload, and lease endpoints require no gateway modifications; +the experimental `.../graft` endpoint requires the dedicated graft support added +in cvmfs PR #4296. --- @@ -1444,7 +1461,7 @@ for manual or infrequent publishes on other repositories. | Layer | Status | |---|---| -| `cvmfs_gateway` | Unmodified; targeted via existing lease-and-payload API | +| `cvmfs_gateway` | Targeted via the existing lease-and-payload API; the DirectGraft fast path additionally requires the `.../graft` endpoint from cvmfs PR #4296 | | Stratum 1 replication daemon | Unmodified (`cvmfs_server snapshot` unchanged) | | Stratum 0 CAS layout | Identical; objects written with the same path structure | | Signed manifest format | Identical; gateway signs as usual | @@ -2131,7 +2148,7 @@ Use this checklist before deploying. Full detail is in Chapter 34. **Publisher (Stratum 0):** - [ ] Recent Go toolchain on the Stratum 0 node -- [ ] `cvmfs_gateway` ≥ 1.2 with the lease-and-payload API +- [ ] `cvmfs_gateway` ≥ 1.2 with the lease-and-payload API (plus the `.../graft` endpoint from cvmfs PR #4296 if using `--gateway-direct-graft`) - [ ] Write access to the CAS backend (local filesystem or S3-compatible) - [ ] Outbound HTTPS from the publisher to the gateway diff --git a/internal/api/orchestrator.go b/internal/api/orchestrator.go index 0d0fc5b..6e7703f 100644 --- a/internal/api/orchestrator.go +++ b/internal/api/orchestrator.go @@ -70,10 +70,12 @@ type Orchestrator struct { Stratum0URL string // DirectGraft enables the fast-path commit on the receiver side. // - // When true, the commit POST body carries "direct_graft":true, instructing - // cvmfs_receiver to skip DiffRec and graft the pre-built subtree catalog - // directly into the parent catalog. This is correct only when the lease - // path is a brand-new directory with no pre-existing content. + // When true, the finalise step POSTs to the dedicated gateway graft + // endpoint (/api/v1/leases//graft) instead of the standard commit + // endpoint, instructing cvmfs_receiver to skip DiffRec and graft the + // pre-built subtree catalog directly into the parent catalog. This is + // correct only when the lease path is a brand-new directory with no + // pre-existing content. // // Set to false (default) to use the standard CommitProcessor/DiffRec path, // which handles arbitrary add/remove/modify operations safely. Both paths diff --git a/internal/lease/backend.go b/internal/lease/backend.go index a45d5d1..c1a7367 100644 --- a/internal/lease/backend.go +++ b/internal/lease/backend.go @@ -62,11 +62,12 @@ type CommitRequest struct { // DirectGraft requests the fast-path commit on the receiver side. // - // When true the commit POST body carries "direct_graft":true, instructing - // the cvmfs_receiver to skip DiffRec entirely and graft the pre-built - // subtree catalog (already uploaded via SubmitPayload) directly into the - // parent catalog. This is correct only when the lease path is a brand-new - // directory with no pre-existing content. + // When true the finalise step POSTs to the dedicated gateway graft endpoint + // /api/v1/leases//graft instead of the standard commit endpoint, + // instructing the cvmfs_receiver to skip DiffRec entirely and graft the + // pre-built subtree catalog (already uploaded via SubmitPayload) directly + // into the parent catalog. This is correct only when the lease path is a + // brand-new directory with no pre-existing content. // // Set to false (the default) to use the standard CommitProcessor / DiffRec // path, which works for arbitrary add/remove/modify operations. Both paths diff --git a/internal/lease/lease.go b/internal/lease/lease.go index 2f32fcf..7103f82 100644 --- a/internal/lease/lease.go +++ b/internal/lease/lease.go @@ -505,10 +505,53 @@ func (c *Client) Heartbeat(ctx context.Context, token string, interval time.Dura // ── Backend interface implementation ───────────────────────────────────────── +// gatewayCommitURL returns the endpoint that finalises a publish transaction. +// +// The standard commit is POST /api/v1/leases/. When req.DirectGraft is +// set the request instead targets the dedicated graft endpoint +// POST /api/v1/leases//graft (gateway/frontend/leases.go +// MakeGraftHandler): the gateway grafts the pre-built subtree catalog into the +// parent, skipping the DiffRec catalog merge. The fast path is selected by the +// endpoint itself, not by a flag in the request body. +func (c *Client) gatewayCommitURL(req CommitRequest) string { + u := fmt.Sprintf("%s/api/v1/leases/%s", c.BaseURL, url.PathEscape(req.Token)) + if req.DirectGraft { + u += "/graft" + } + return u +} + +// gatewayCommitBody marshals the JSON body shared by the commit and graft +// endpoints. Both decode an identical body — catalog root hashes plus the +// optional snapshot tag — so no "direct_graft" field is sent; the endpoint +// (see gatewayCommitURL) selects the DirectGraft fast path. +func gatewayCommitBody(req CommitRequest) ([]byte, error) { + // Use NewRootHashSuffixed if provided; otherwise fall back to CatalogHash + // for backward compatibility. + newHash := req.NewRootHashSuffixed + if newHash == "" { + newHash = req.CatalogHash + } + + body, err := json.Marshal(map[string]interface{}{ + "old_root_hash": req.OldRootHash, + "new_root_hash": newHash, + "tag_name": req.TagName, + "tag_description": req.TagDescription, + }) + if err != nil { + return nil, fmt.Errorf("marshalling commit body: %w", err) + } + return body, nil +} + // Commit implements Backend for the gateway mode. It uploads all staged CAS // objects to the gateway via SubmitPayload, then finalises the publish by // POSTing to /api/v1/leases/ — distinct from DELETE which cancels. // +// When req.DirectGraft is set the finalise step instead POSTs to the dedicated +// graft endpoint /api/v1/leases//graft; see gatewayCommitURL. +// // Per gateway/frontend/leases.go: // // POST /api/v1/leases/ → handleCommitLease (finalise/publish) @@ -534,25 +577,14 @@ func (c *Client) Commit(ctx context.Context, req CommitRequest) error { return fmt.Errorf("gateway submit payload: %w", err) } - // 2. POST /api/v1/leases/ to finalise the publish transaction. - commitURL := fmt.Sprintf("%s/api/v1/leases/%s", c.BaseURL, url.PathEscape(req.Token)) - - // Use NewRootHashSuffixed if provided; otherwise fall back to CatalogHash for backward compatibility. - newHash := req.NewRootHashSuffixed - if newHash == "" { - newHash = req.CatalogHash - } + // 2. POST the finalise request. DirectGraft targets the dedicated graft + // endpoint; otherwise the standard commit endpoint (see gatewayCommitURL). + commitURL := c.gatewayCommitURL(req) - commitBody, err := json.Marshal(map[string]interface{}{ - "old_root_hash": req.OldRootHash, - "new_root_hash": newHash, - "tag_name": req.TagName, - "tag_description": req.TagDescription, - "direct_graft": req.DirectGraft, - }) + commitBody, err := gatewayCommitBody(req) if err != nil { span.RecordError(err) - return fmt.Errorf("marshalling commit body: %w", err) + return err } postReq, err := http.NewRequestWithContext(ctx, "POST", commitURL, bytes.NewReader(commitBody)) @@ -614,23 +646,12 @@ func (c *Client) CommitFinalizeOnly(ctx context.Context, req CommitRequest) erro ctx, span := c.obs.Tracer.Start(ctx, "lease.commit_finalize_only") defer span.End() - commitURL := fmt.Sprintf("%s/api/v1/leases/%s", c.BaseURL, url.PathEscape(req.Token)) - - newHash := req.NewRootHashSuffixed - if newHash == "" { - newHash = req.CatalogHash - } + commitURL := c.gatewayCommitURL(req) - commitBody, err := json.Marshal(map[string]interface{}{ - "old_root_hash": req.OldRootHash, - "new_root_hash": newHash, - "tag_name": req.TagName, - "tag_description": req.TagDescription, - "direct_graft": req.DirectGraft, - }) + commitBody, err := gatewayCommitBody(req) if err != nil { span.RecordError(err) - return fmt.Errorf("marshalling commit body: %w", err) + return err } postReq, err := http.NewRequestWithContext(ctx, "POST", commitURL, bytes.NewReader(commitBody)) diff --git a/internal/lease/lease_test.go b/internal/lease/lease_test.go index a9bfd8f..cfca5e2 100644 --- a/internal/lease/lease_test.go +++ b/internal/lease/lease_test.go @@ -461,6 +461,84 @@ func TestCommit_SendsTagFields(t *testing.T) { } } +// TestCommit_DirectGraftRouting verifies that DirectGraft selects the dedicated +// graft endpoint (POST /api/v1/leases//graft) while a plain commit uses +// POST /api/v1/leases/, and that no "direct_graft" flag is sent in the +// body (the endpoint, not a body field, selects the fast path). +func TestCommit_DirectGraftRouting(t *testing.T) { + const ( + token = "lease-tok-graft" + catalogHash = "feedface" + ) + + for _, tc := range []struct { + name string + directGraft bool + wantPath string + }{ + {"plain commit", false, "/api/v1/leases/" + token}, + {"direct graft", true, "/api/v1/leases/" + token + "/graft"}, + } { + t.Run(tc.name, func(t *testing.T) { + var ( + mu sync.Mutex + capturedPath string + capturedBody []byte + ) + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/payloads": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok"}`)) + case r.Method == http.MethodPost && strings.HasPrefix(r.URL.Path, "/api/v1/leases/"): + data, _ := io.ReadAll(r.Body) + mu.Lock() + capturedPath = r.URL.Path + capturedBody = data + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"status":"ok"}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + + c := newTestClient(t, srv) + store := &mockObjectReader{ + objects: map[string][]byte{catalogHash: []byte("fake-compressed-catalog")}, + } + + req := CommitRequest{ + Token: token, + CatalogHash: catalogHash, + ObjectStore: store, + DirectGraft: tc.directGraft, + } + if err := c.Commit(context.Background(), req); err != nil { + t.Fatalf("Commit: %v", err) + } + + mu.Lock() + path := capturedPath + body := capturedBody + mu.Unlock() + + if path != tc.wantPath { + t.Errorf("commit POST path = %q; want %q", path, tc.wantPath) + } + var got map[string]interface{} + if err := json.Unmarshal(body, &got); err != nil { + t.Fatalf("decode commit body %q: %v", body, err) + } + if _, ok := got["direct_graft"]; ok { + t.Errorf("commit body must not carry a direct_graft flag; got %q", body) + } + }) + } +} + // TestCommit_EmptyTagName checks that when TagName is empty the commit body // still contains a "tag_name" key (empty string) so the gateway receives a // well-formed request. From 0c02df4f523c8e1ea1f4085e88d71b5bfdceca98 Mon Sep 17 00:00:00 2001 From: Valentin Volkl Date: Sat, 4 Jul 2026 18:49:06 +0200 Subject: [PATCH 2/2] add ci --- .github/workflows/gateway-publish.yml | 72 +++++++ test/integration/gateway/README.md | 82 ++++++++ test/integration/gateway/docker-compose.yml | 150 ++++++++++++++ test/integration/gateway/run.sh | 211 ++++++++++++++++++++ 4 files changed, 515 insertions(+) create mode 100644 .github/workflows/gateway-publish.yml create mode 100644 test/integration/gateway/README.md create mode 100644 test/integration/gateway/docker-compose.yml create mode 100755 test/integration/gateway/run.sh diff --git a/.github/workflows/gateway-publish.yml b/.github/workflows/gateway-publish.yml new file mode 100644 index 0000000..6139e42 --- /dev/null +++ b/.github/workflows/gateway-publish.yml @@ -0,0 +1,72 @@ +# Integration test: publish through a real cvmfs_gateway. +# +# Builds a mountless cvmfs_gateway + cvmfs_receiver from cvmfs@devel, backed by +# S3 (Garage), and drives a full publish through cvmfs-prepub in gateway mode. +# This is the only test that exercises the live lease → payload → commit/graft +# path against the actual gateway; the Go unit tests only cover the client side. +# +# It is heavy (builds cvmfs from source), so it runs on demand and on pull +# requests that touch the gateway client or the fixtures — not on every push. + +name: gateway-publish + +on: + workflow_dispatch: + inputs: + cvmfs_ref: + description: "cvmfs git ref to build the gateway from" + required: false + default: "devel" + pull_request: + paths: + - "internal/lease/**" + - "internal/api/**" + - "cmd/prepub/**" + - "test/integration/gateway/**" + - ".github/workflows/gateway-publish.yml" + push: + branches: + - "gateway-dedicated-graft-endpoint-test" + +# A gateway build from source is expensive; don't pile up concurrent runs. +concurrency: + group: gateway-publish-${{ github.ref }} + cancel-in-progress: true + +jobs: + gateway-publish: + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + CVMFS_REF: ${{ github.event.inputs.cvmfs_ref || 'devel' }} + steps: + - name: Check out cvmfs-bits + uses: actions/checkout@v4 + with: + path: cvmfs-bits + + - name: Check out cvmfs (${{ env.CVMFS_REF }}) + uses: actions/checkout@v4 + with: + repository: cvmfs/cvmfs + ref: ${{ env.CVMFS_REF }} + path: cvmfs + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: cvmfs-bits/go.mod + cache-dependency-path: cvmfs-bits/go.sum + + - name: Run gateway publish integration test + working-directory: cvmfs-bits + env: + CVMFS_SRC: ${{ github.workspace }}/cvmfs + run: ./test/integration/gateway/run.sh + + - name: Dump gateway logs on failure + if: failure() + working-directory: cvmfs-bits + env: + CVMFS_SRC: ${{ github.workspace }}/cvmfs + run: docker compose -f test/integration/gateway/docker-compose.yml logs --no-color || true diff --git a/test/integration/gateway/README.md b/test/integration/gateway/README.md new file mode 100644 index 0000000..1d4b211 --- /dev/null +++ b/test/integration/gateway/README.md @@ -0,0 +1,82 @@ +# Gateway publish integration test + +End-to-end test that exercises `cvmfs-prepub` publishing through a **real** +`cvmfs_gateway` + `cvmfs_receiver`, backed by S3 (Garage). The Go unit tests +only cover the client side of the lease/payload/commit API; this is the only +test that drives the live path against the actual gateway, including the +DirectGraft fast-path commit. + +## What it brings up + +The stack combines two fixtures from the cvmfs source tree: + +- the S3 backend (Garage) from `test/common/container/s3-integration` +- a **mountless** gateway (`cvmfs_server mkfs -P -D` — no FUSE, no systemd, no + privileged) from `test/common/container/publish-mountless`, built from + `cvmfs@devel` + +There is **no** `cvmfs_server` publisher container: the publisher is +`cvmfs-prepub`, which speaks the gateway lease/payload/commit API directly. +`prepub` runs on the host and reaches the gateway on the published port `4929`. + +``` + cvmfs-prepub (host, gateway mode) + │ lease → payload → commit/graft (HMAC-signed HTTP :4929) + │ ▲ reads .cvmfspublished for old_root_hash (web endpoint :3902) + ▼ │ + cvmfs_gateway ── spawns ──▶ cvmfs_receiver (mountless container) + │ writes objects/catalogs + ▼ + Garage (S3) ◀── clients read repo data (web endpoint :3902) +``` + +`prepub` is started with `--stratum0-url http://localhost:3902` (Garage's web +endpoint). This is **required**: prepub only builds the subtree catalog — the +one that yields `new_root_hash` — when a stratum0 URL is configured. Without it +the commit carries a null `new_root_hash` and the receiver rejects the +DirectGraft with `merge_error` / "DirectGraft requires a catalog hash". For a +fresh repo the manifest GET returns 404 (Garage routes buckets by Host header), +which prepub treats as "first publish" (empty `old_root_hash`); the receiver +fetches the real base manifest itself, so that is correct here. + +## Running locally + +Requires Docker (with Compose v2), Go, and a cvmfs checkout on `devel`: + +```sh +CVMFS_SRC=/path/to/cvmfs ./run.sh +``` + +The first run builds the gateway image from cvmfs source, which is slow. Set +`KEEP_UP=1` to leave the stack and `prepub` running afterwards for debugging: + +```sh +KEEP_UP=1 CVMFS_SRC=/path/to/cvmfs ./run.sh +# ... poke at http://localhost:4929 / http://localhost:8080 ... +CVMFS_SRC=/path/to/cvmfs docker compose -f docker-compose.yml down -v +``` + +## In CI + +`.github/workflows/gateway-publish.yml` checks out both repos, then runs +`run.sh`. It triggers on `workflow_dispatch` (with an optional `cvmfs_ref` +input) and on pull requests that touch the gateway client (`internal/lease`, +`internal/api`, `cmd/prepub`) or these fixtures. It does **not** run on every +push — building cvmfs from source is expensive. + +## Credentials + +All dev/test only, and must stay in sync across the pieces: + +| Secret | Where | Value | +| --- | --- | --- | +| S3 access/secret | `docker-compose.yml` ↔ `garage-setup` ↔ gateway | fixed in compose | +| Gateway lease key | gateway entrypoint writes it to `/etc/cvmfs/keys/.gw`; prepub signs with the same `CVMFS_GATEWAY_KEY_ID`/`CVMFS_GATEWAY_SECRET` in `run.sh` | `mykey` / `mysecret` | +| prepub API token | `run.sh` `PREPUB_API_TOKEN` | `integration-test-token` | + +> The gateway only accepts a lease key that its **access config** associates +> with the repository. The gateway image bakes in `gateway/config/repo.json`, +> which only knows `example_repo.domain.org`, so `docker-compose.yml` mounts the +> mountless `config/repo.json` (which lists `test.repo.org`) and `config/user.json` +> (`enable_key_endpoint`) over it. Without that mount every lease is rejected +> with `invalid key ID specified`, regardless of the key value. diff --git a/test/integration/gateway/docker-compose.yml b/test/integration/gateway/docker-compose.yml new file mode 100644 index 0000000..eb913b3 --- /dev/null +++ b/test/integration/gateway/docker-compose.yml @@ -0,0 +1,150 @@ +# docker-compose.yml – cvmfs-bits ↔ cvmfs_gateway integration stack +# +# Brings up the minimum needed to exercise cvmfs-prepub publishing against a +# real cvmfs_gateway + cvmfs_receiver, backed by S3 (Garage). It combines two +# fixtures from the cvmfs source tree (checked out at $CVMFS_SRC): +# +# • the S3 backend from test/common/container/s3-integration (Garage) +# • a MOUNTLESS gateway test/common/container/publish-mountless (no FUSE, +# no systemd, no privileged) built from cvmfs@devel +# +# Unlike publish-mountless, there is NO cvmfs publisher container here: the +# publisher is cvmfs-prepub, which talks the gateway lease/payload/commit API +# directly over HTTP. prepub runs on the host (see run.sh) and reaches the +# gateway via the published port 4929. +# +# Services +# -------- +# garage – Garage S3-compatible object storage (v2) +# garage-setup – one-shot init: layout, key, bucket, website access +# gateway – cvmfs_gateway + cvmfs_receiver (mountless, unprivileged) +# +# Requires the environment variable CVMFS_SRC to point at a cvmfs checkout +# (devel branch) so the gateway image and Garage fixtures can be built/mounted. +# +# Usage +# ----- +# CVMFS_SRC=/path/to/cvmfs docker compose up --build -d +# # gateway API → http://localhost:4929 garage web → http://localhost:3902 +# CVMFS_SRC=/path/to/cvmfs docker compose down -v --remove-orphans + +name: cvmfs-bits-gateway + +# --------------------------------------------------------------------------- +# Fixed credentials – dev/test only. The S3 creds must match between +# garage-setup and gateway. GW_KEY_ID/GW_KEY_SECRET are the gateway lease key: +# the entrypoint writes them to /etc/cvmfs/keys/.gw, and repo.json (see +# the gateway volume mounts) associates that "default" key with test.repo.org. +# prepub must sign with the same id/secret (see run.sh). +# --------------------------------------------------------------------------- +x-s3-env: &s3-env + S3_ACCESS_KEY: GK00c4f5e0a1b2c3d4e5f60011 + S3_SECRET_KEY: 00c4f5e0a1b2c3d4e5f6001100c4f5e0a1b2c3d4e5f6001100c4f5e0a1b2c3d4 + S3_BUCKET: cvmfs + +x-gw-key-env: &gw-key-env + GW_KEY_ID: mykey + GW_KEY_SECRET: mysecret + +services: + + # ----------------------------------------------------------------------- + # Garage – S3-compatible object storage (v2) + # + # The network alias cvmfs.web.garage.internal lets the gateway (and Garage + # itself) map the .web.garage.internal Host header to this + # container, so the Stratum-0 URL resolves inside the compose network. + # ----------------------------------------------------------------------- + garage: + image: docker.io/dxflrs/garage:v2.2.0 + container_name: cvmfs-bits-garage + hostname: cvmfs-garage + networks: + default: + aliases: + - cvmfs.web.garage.internal + volumes: + - ${CVMFS_SRC:?set CVMFS_SRC to a cvmfs checkout}/test/common/container/publish-mountless/config/garage.toml:/etc/garage.toml:ro + - garage-meta:/var/lib/garage/meta + - garage-data:/var/lib/garage/data + ports: + - "3902:3902" # Web / website access (anonymous reads, verification) + healthcheck: + test: ["CMD", "/garage", "node", "id"] + interval: 3s + timeout: 5s + retries: 10 + + # ----------------------------------------------------------------------- + # Garage setup – runs once after Garage is healthy + # ----------------------------------------------------------------------- + garage-setup: + image: alpine:3 + container_name: cvmfs-bits-garage-setup + depends_on: + garage: + condition: service_healthy + environment: + GARAGE_ADMIN_URL: http://cvmfs-garage:3903 + GARAGE_ADMIN_TOKEN: garage-admin-token + SETUP_GARAGE_DEBUG: "1" + <<: *s3-env + volumes: + - ${CVMFS_SRC:?set CVMFS_SRC to a cvmfs checkout}/test/common/container/publish-mountless/scripts/setup_garage.sh:/setup_garage.sh:ro + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -e + echo '[garage-setup] Installing curl, jq, openssl ...' + apk add --no-cache curl jq openssl + echo '[garage-setup] Running setup_garage.sh ...' + sh /setup_garage.sh + restart: "no" + + # ----------------------------------------------------------------------- + # Gateway + Receiver (mountless – no FUSE, no privileged, no systemd) + # + # Built from the cvmfs@devel source tree. The entrypoint runs + # `cvmfs_server mkfs -P -D` on first boot to initialise test.repo.org on + # the S3 backend, writes the gateway lease key, then starts cvmfs_gateway. + # ----------------------------------------------------------------------- + gateway: + build: + context: ${CVMFS_SRC:?set CVMFS_SRC to a cvmfs checkout} + dockerfile: test/common/container/publish-mountless/Dockerfile.gateway + image: cvmfs-gateway-mountless:latest + container_name: cvmfs-bits-gateway + hostname: cvmfs-gateway + depends_on: + garage-setup: + condition: service_completed_successfully + environment: + REPO_NAME: test.repo.org + REPO_OWNER: root + GARAGE_HOST: cvmfs-garage + GARAGE_S3_PORT: "3900" + # Stratum-0 URL points at Garage's web endpoint via the network + # alias, so the receiver can fetch existing catalogs on commit. + STRATUM0_URL: http://cvmfs.web.garage.internal:3902 + <<: [*gw-key-env, *s3-env] + volumes: + # Gateway access config: repo.json must list test.repo.org (as a + # "default"-keyed repo) or the gateway won't know the repository and + # rejects every lease with "invalid key ID specified". user.json + # sets enable_key_endpoint so the repo signing keys can be fetched. + # The image bakes in gateway/config/{repo,user}.json, which only + # know example_repo.domain.org, so we overlay the mountless ones. + - ${CVMFS_SRC:?set CVMFS_SRC to a cvmfs checkout}/test/common/container/publish-mountless/config/repo.json:/etc/cvmfs/gateway/repo.json:ro + - ${CVMFS_SRC:?set CVMFS_SRC to a cvmfs checkout}/test/common/container/publish-mountless/config/user.json:/etc/cvmfs/gateway/user.json:ro + - etc-cvmfs-repos:/etc/cvmfs/repositories.d + - etc-cvmfs-keys:/etc/cvmfs/keys + - var-lib-gateway:/var/lib/cvmfs-gateway + ports: + - "4929:4929" # Gateway lease API (prepub connects here) + +volumes: + garage-meta: + garage-data: + etc-cvmfs-repos: + etc-cvmfs-keys: + var-lib-gateway: diff --git a/test/integration/gateway/run.sh b/test/integration/gateway/run.sh new file mode 100755 index 0000000..89a6bc4 --- /dev/null +++ b/test/integration/gateway/run.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# run.sh – end-to-end gateway publish integration test for cvmfs-bits. +# +# Brings up a mountless cvmfs_gateway backed by S3 (Garage) via +# docker-compose.yml, then drives a full publish through cvmfs-prepub in +# gateway mode: +# +# 1. build + start the Garage + mountless-gateway stack (needs CVMFS_SRC) +# 2. build cvmfs-prepub and run it on the host in gateway mode +# 3. submit a publish job (a small tar) via the prepub HTTP API +# 4. poll the job to a terminal state; require "published" +# 5. confirm the repository's .cvmfspublished is served from the S3 backend +# +# The publisher is cvmfs-prepub itself (it speaks the gateway lease/payload/ +# commit API directly) — there is no cvmfs_server publisher container. +# +# Environment +# ----------- +# CVMFS_SRC (required) path to a cvmfs checkout on the devel branch +# KEEP_UP (optional) if set to 1, leave the stack + prepub running on exit +# +# Usage +# ----- +# CVMFS_SRC=/path/to/cvmfs ./run.sh + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration (matches docker-compose.yml) +# --------------------------------------------------------------------------- +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${HERE}/../../.." && pwd)" + +REPO_NAME="test.repo.org" +PUBLISH_PATH="hello" # a brand-new subtree → direct-graft valid +GATEWAY_URL="http://localhost:4929" +GARAGE_WEB="http://localhost:3902" +GARAGE_WEB_HOST="cvmfs.web.garage.internal" + +# Gateway lease key. The gateway entrypoint writes "plain_text mykey mysecret" +# to /etc/cvmfs/keys/.gw and the mounted repo.json associates that key +# with test.repo.org, so prepub must sign with the same id/secret. (These are +# fixed dev/test credentials, matching docker-compose.yml's GW_KEY_* env.) +export CVMFS_GATEWAY_KEY_ID="mykey" +export CVMFS_GATEWAY_SECRET="mysecret" + +# Bearer token guarding the prepub HTTP API. +export PREPUB_API_TOKEN="integration-test-token" + +PREPUB_LISTEN="127.0.0.1:8080" +PREPUB_API="http://${PREPUB_LISTEN}" + +: "${CVMFS_SRC:?set CVMFS_SRC to a cvmfs checkout (devel branch)}" +export CVMFS_SRC + +COMPOSE=(docker compose -f "${HERE}/docker-compose.yml") + +WORKDIR="$(mktemp -d)" +PREPUB_PID="" + +log() { printf '\n\033[1;34m[run.sh]\033[0m %s\n' "$*"; } +err() { printf '\n\033[1;31m[run.sh] ERROR:\033[0m %s\n' "$*" >&2; } + +# --------------------------------------------------------------------------- +# Teardown – always dump gateway logs and tear the stack down (unless KEEP_UP) +# --------------------------------------------------------------------------- +cleanup() { + local rc=$? + if [[ -n "${PREPUB_PID}" ]] && kill -0 "${PREPUB_PID}" 2>/dev/null; then + kill "${PREPUB_PID}" 2>/dev/null || true + wait "${PREPUB_PID}" 2>/dev/null || true + fi + if [[ "${rc}" -ne 0 ]]; then + err "failed (exit ${rc}) — dumping diagnostics" + [[ -f "${WORKDIR}/prepub.log" ]] && { echo "──── prepub.log ────"; tail -n 100 "${WORKDIR}/prepub.log"; } + echo "──── gateway logs ────"; "${COMPOSE[@]}" logs --no-color --tail 100 gateway 2>/dev/null || true + fi + if [[ "${KEEP_UP:-0}" != "1" ]]; then + log "tearing down stack" + "${COMPOSE[@]}" down -v --remove-orphans 2>/dev/null || true + rm -rf "${WORKDIR}" + else + log "KEEP_UP=1 — leaving stack up; artifacts in ${WORKDIR}" + fi + exit "${rc}" +} +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Poll a URL until it responds (2xx/4xx) or times out. +# --------------------------------------------------------------------------- +wait_for_http() { + local url="$1" name="$2" tries="${3:-60}" + log "waiting for ${name} at ${url}" + for ((i = 0; i < tries; i++)); do + if curl -s -o /dev/null --max-time 3 "${url}"; then + log "${name} is up" + return 0 + fi + sleep 2 + done + err "${name} did not become ready at ${url}" + return 1 +} + +# --------------------------------------------------------------------------- +# 1. Build + start the gateway stack +# --------------------------------------------------------------------------- +log "building + starting Garage + mountless gateway (CVMFS_SRC=${CVMFS_SRC})" +"${COMPOSE[@]}" up --build -d + +# The gateway entrypoint runs `cvmfs_server mkfs` on first boot, which takes a +# while; poll the lease API root until it answers. +wait_for_http "${GATEWAY_URL}/api/v1" "gateway lease API" 120 + +# --------------------------------------------------------------------------- +# 2. Build + start cvmfs-prepub (gateway mode) on the host +# --------------------------------------------------------------------------- +log "building cvmfs-prepub" +( cd "${REPO_ROOT}" && go build -o "${WORKDIR}/cvmfs-prepub" ./cmd/prepub ) + +mkdir -p "${WORKDIR}/spool" "${WORKDIR}/cas" + +log "starting cvmfs-prepub against ${GATEWAY_URL}" +# --stratum0-url is REQUIRED for gateway publishing: prepub only builds the +# subtree catalog (BuildSubtree, which produces new_root_hash) when a stratum0 +# URL is configured. Without it the commit sends a null new_root_hash and the +# receiver rejects the DirectGraft with "merge_error" ("DirectGraft requires a +# catalog hash"). We point it at Garage's web endpoint on localhost; because +# Garage routes buckets by Host header, a bare localhost request for a fresh +# repo returns 404, which prepub correctly treats as "first publish, no existing +# manifest" (empty old_root_hash) — exactly right here. The receiver fetches +# the real base manifest itself over the compose network, so an empty +# old_root_hash does not affect the graft. +"${WORKDIR}/cvmfs-prepub" \ + --dev \ + --publish-mode gateway \ + --gateway-url "${GATEWAY_URL}" \ + --gateway-direct-graft=true \ + --stratum0-url "${GARAGE_WEB}" \ + --listen "${PREPUB_LISTEN}" \ + --spool-root "${WORKDIR}/spool" \ + --cas-type localfs \ + --cas-root "${WORKDIR}/cas" \ + --repo-name "${REPO_NAME}" \ + > "${WORKDIR}/prepub.log" 2>&1 & +PREPUB_PID=$! + +wait_for_http "${PREPUB_API}/api/v1/health" "prepub API" 30 + +# --------------------------------------------------------------------------- +# 3. Submit a publish job (a small tar at a brand-new subtree) +# --------------------------------------------------------------------------- +log "building payload tar" +mkdir -p "${WORKDIR}/payload/${PUBLISH_PATH}" +echo "hello from cvmfs-bits gateway integration test" \ + > "${WORKDIR}/payload/${PUBLISH_PATH}/greeting.txt" +tar -C "${WORKDIR}/payload/${PUBLISH_PATH}" -cf "${WORKDIR}/payload.tar" . + +log "submitting publish job (repo=${REPO_NAME} path=${PUBLISH_PATH})" +submit_resp="$(curl -sf \ + -H "Authorization: Bearer ${PREPUB_API_TOKEN}" \ + -F "repo=${REPO_NAME}" \ + -F "path=${PUBLISH_PATH}" \ + -F "tar=@${WORKDIR}/payload.tar" \ + "${PREPUB_API}/api/v1/jobs")" +echo "submit response: ${submit_resp}" + +job_id="$(printf '%s' "${submit_resp}" | sed -n 's/.*"job_id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')" +[[ -n "${job_id}" ]] || { err "no job_id in submit response"; exit 1; } +log "job id: ${job_id}" + +# --------------------------------------------------------------------------- +# 4. Poll the job to a terminal state +# --------------------------------------------------------------------------- +log "polling job ${job_id}" +state="" +for ((i = 0; i < 90; i++)); do + job_json="$(curl -sf -H "Authorization: Bearer ${PREPUB_API_TOKEN}" \ + "${PREPUB_API}/api/v1/jobs/${job_id}" || true)" + state="$(printf '%s' "${job_json}" | sed -n 's/.*"state"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')" + case "${state}" in + published) + log "job published" + echo "${job_json}" + break + ;; + failed|aborted) + err "job reached terminal state '${state}'" + echo "${job_json}" + exit 1 + ;; + esac + sleep 2 +done +[[ "${state}" == "published" ]] || { err "job did not publish (last state='${state:-none}')"; exit 1; } + +# --------------------------------------------------------------------------- +# 5. Confirm the published manifest is readable from the S3 backend +# --------------------------------------------------------------------------- +log "verifying .cvmfspublished on the S3 backend" +if curl -sf --max-time 10 -H "Host: ${GARAGE_WEB_HOST}" \ + "${GARAGE_WEB}/${REPO_NAME}/.cvmfspublished" -o "${WORKDIR}/cvmfspublished"; then + root_hash="$(sed -n 's/^C\(.*\)/\1/p' "${WORKDIR}/cvmfspublished" | head -1)" + log "published root catalog hash: ${root_hash:-}" +else + err ".cvmfspublished not readable from ${GARAGE_WEB}/${REPO_NAME}/" + exit 1 +fi + +log "SUCCESS: cvmfs-prepub published ${REPO_NAME}:/${PUBLISH_PATH} through the gateway"