feat(loadtest): add loadtest-app — publisher + mock receiver with dashboard

.gitignore

@@ -43,3 +43,6 @@ skills-lock.json
 *.local.*
 bin/
 .envrc
+loadtest/app/test-results/
+
+loadtest/app/tmp/

Makefile

@@ -49,6 +49,32 @@ health:
 smoke:
 	./build/dev/smoke.sh
 
+up/loadtest:
+	docker-compose -f build/dev/loadtest/compose.yml up -d --build
+
+down/loadtest:
+	docker-compose -f build/dev/loadtest/compose.yml down
+
+# Benchmark stack: outpost + loadtest app + prometheus + grafana.
+# Prometheus is on :9091 (the loadtest app owns :9090), Grafana on :3000.
+up/bench:
+	LOCAL_DEV_GRAFANA=1 ./build/dev/dev.sh up
+	docker-compose -f build/dev/loadtest/compose.yml up -d --build
+
+down/bench:
+	docker-compose -f build/dev/loadtest/compose.yml down
+	LOCAL_DEV_GRAFANA=1 ./build/dev/dev.sh down
+
+# One benchmark, end to end: validate → run → export → figures.
+#
+# The default spec is the committed example — a four-minute pipeline check, not
+# a result. Point SPEC at your own to run a real one; specs are client-side
+# input and are not kept in the repo. Drive a deployed app with LOADTEST_URL
+# and PROM_URL, and see build/dev/bench.sh for --detach / --report.
+SPEC ?= loadtest/app/runs/example.yaml
+bench:
+	./build/dev/bench.sh $(SPEC)
+
 # Run portal natively (vite hot reload). Portal is also available as a
 # containerized service via `make up` — use this target only when you want
 # the faster native dev loop.

build/dev/bench.sh

@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+# Runs one benchmark end to end: spec → run → Prometheus → export → figures.
+#
+# Nothing here is interactive and nothing needs Grafana. Grafana is for
+# watching a run in progress; the artifact comes out of this script.
+#
+# The spec is client-side input: it is read here and POSTed to the app, so the
+# deployment never needs a copy. Everything the script touches on the app goes
+# over HTTP, which is what lets the same command drive a local stack or a
+# deployed one:
+#
+#   bench.sh                            # the example spec against localhost
+#   bench.sh path/to/spec.yaml
+#   LOADTEST_URL=https://… PROM_URL=https://… bench.sh path/to/spec.yaml
+#   bench.sh --detach path/to/spec.yaml # start it and exit; render later
+#   bench.sh --report <run_id>          # render a run that already finished
+#
+# --detach exists because a long run should not need the operator's laptop
+# awake for its whole window. Start it, walk away, report on it afterwards.
+set -euo pipefail
+
+APP="${LOADTEST_URL:-http://localhost:9090}"
+PROM="${PROM_URL:-http://localhost:9091}"
+OUT="${BENCH_OUT:-loadtest/out}"
+REPORT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/loadtest/report"
+
+DETACH=0
+REPORT_ONLY=""
+SPEC=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --detach) DETACH=1; shift ;;
+    --report) REPORT_ONLY="${2:-}"; shift 2 ;;
+    -*) echo "error: unknown flag $1" >&2; exit 2 ;;
+    *) SPEC="$1"; shift ;;
+  esac
+done
+SPEC="${SPEC:-loadtest/app/runs/example.yaml}"
+
+need() { command -v "$1" >/dev/null || { echo "error: $1 is required" >&2; exit 2; }; }
+need curl
+need jq
+need python3
+
+# Auth for a deployed app. Local dev has none, so the header is omitted when
+# the variable is unset rather than sent empty.
+AUTH=()
+if [ -n "${LOADTEST_TOKEN:-}" ]; then
+  AUTH=(-H "Authorization: Bearer $LOADTEST_TOKEN")
+fi
+
+# --- report: fetch the artifact over HTTP, enrich it, render it -------------
+# Fetching rather than reading the filesystem is what makes a remote target
+# work: the app writes the export into its own container, which the operator's
+# machine cannot see.
+report() {
+  local run_id="$1"
+  # Retrying only makes sense straight after a run, when the export may still
+  # be being written. Asked for a run that finished long ago — or a typo — the
+  # first 404 is the answer, and waiting half a minute for it is just noise.
+  local attempts="${2:-1}"
+  mkdir -p "$OUT"
+  local artifact="$OUT/$run_id.json"
+
+  local ok=0
+  for _ in $(seq 1 "$attempts"); do
+    if curl -fsS "${AUTH[@]}" "$APP/api/runs/$run_id" -o "$artifact.tmp" 2>/dev/null \
+       && jq -e '.run.id' "$artifact.tmp" >/dev/null 2>&1; then
+      mv "$artifact.tmp" "$artifact"; ok=1; break
+    fi
+    sleep 2
+  done
+  rm -f "$artifact.tmp"
+  if [ "$ok" != "1" ]; then
+    echo "error: no export available for $run_id at $APP" >&2
+    exit 1
+  fi
+
+  local voids
+  voids="$(jq -r '.run.voids // [] | length' "$artifact")"
+  if [ "$voids" != "0" ]; then
+    echo "==> run is VOID:"
+    jq -r '.run.voids[] | "    - \(.)"' "$artifact"
+  fi
+
+  echo "==> fetching series from prometheus"
+  python3 "$REPORT_DIR/fetch.py" --prom "$PROM" --artifact "$artifact" --out "$artifact"
+
+  # Raw capture is separate from the enriched export and much larger: full
+  # native-histogram buckets at the scrape interval, over the whole run rather
+  # than the steady window. It is what makes a question that occurs to someone
+  # next month answerable without paying for the run again.
+  echo "==> archiving raw series"
+  python3 "$REPORT_DIR/capture.py" --prom "$PROM" --artifact "$artifact" \
+    --out "${artifact%.json}" || echo "warning: raw capture failed" >&2
+
+  echo "==> rendering figures"
+  python3 "$REPORT_DIR/make_charts.py" --data "$artifact"
+
+  jq -r '"==> " + .run.id,
+         "    published \(.total.published)  completed \(.total.completed)  missing \(.total.missing)  cutoff \(.total.cutoff)",
+         "    export    '"$artifact"'"' "$artifact"
+  echo "    figures   $REPORT_DIR/charts/"
+  [ "$voids" != "0" ] && echo "    VOID — figures carry the void notice; do not publish" && exit 3
+  exit 0
+}
+
+if [ -n "$REPORT_ONLY" ]; then
+  report "$REPORT_ONLY"
+fi
+
+if [ ! -f "$SPEC" ]; then
+  echo "error: spec not found: $SPEC" >&2
+  exit 2
+fi
+
+# --- preflight -------------------------------------------------------------
+if ! curl -fsS "${AUTH[@]}" "$APP/api/status" >/dev/null 2>&1; then
+  echo "error: loadtest app not reachable at $APP — run 'make up/bench' first" >&2
+  exit 1
+fi
+if ! curl -fsS "$PROM/-/ready" >/dev/null 2>&1; then
+  echo "error: prometheus not reachable at $PROM — run 'make up/bench' first" >&2
+  exit 1
+fi
+
+# Fail before provisioning rather than after: a spec that busts its budget
+# measures saturation, and finding that out an hour in is expensive.
+echo "==> validating $SPEC"
+validation="$(curl -sS "${AUTH[@]}" -X POST "$APP/api/runs/validate" --data-binary @"$SPEC")"
+if [ "$(jq -r '.valid' <<<"$validation")" != "true" ]; then
+  jq -r '.error' <<<"$validation" >&2
+  exit 1
+fi
+jq -r '.budget | "    \(.offered_rate) events/s · \(.concurrency|floor) concurrent deliveries · \(.bytes_per_sec/1000000*100|floor/100) MB/s"' <<<"$validation"
+
+# --- run -------------------------------------------------------------------
+echo "==> starting run"
+started="$(curl -sS "${AUTH[@]}" -X POST "$APP/api/runs" --data-binary @"$SPEC")"
+RUN_ID="$(jq -r '.id' <<<"$started")"
+if [ "$RUN_ID" = "null" ] || [ -z "$RUN_ID" ]; then
+  echo "$started" >&2
+  exit 1
+fi
+echo "    run_id=$RUN_ID"
+
+if [ "$DETACH" = "1" ]; then
+  cat <<EOF
+    detached — the run continues on the app.
+    watch:   curl -s $APP/api/runs/current | jq -r .phase
+    report:  $0 --report $RUN_ID
+EOF
+  exit 0
+fi
+
+# Abort the run if the operator interrupts, so a killed script doesn't leave
+# publishers hammering the deployment.
+trap 'echo; echo "==> aborting run"; curl -sS "${AUTH[@]}" -X POST "$APP/api/runs/current/abort" >/dev/null || true; exit 130' INT TERM
+
+phase=""
+while true; do
+  cur="$(curl -sS "${AUTH[@]}" "$APP/api/runs/current")"
+  p="$(jq -r '.phase' <<<"$cur")"
+  if [ "$p" != "$phase" ]; then
+    printf '    %-9s %s\n' "$p" "$(date +%H:%M:%S)"
+    phase="$p"
+  fi
+  case "$p" in
+    complete) break ;;
+    failed|aborted)
+      jq -r '.error // "run did not complete"' <<<"$cur" >&2
+      exit 1
+      ;;
+  esac
+  sleep 5
+done
+
+report "$RUN_ID" 15

build/dev/grafana/compose.yml

@@ -1,11 +1,16 @@
 name: "outpost"
 
+# Host paths are relative to build/dev/, not to this file. dev.sh always passes
+# build/dev/compose.yml first, which makes that the compose project directory,
+# and volume sources resolve against it. A `./foo.yml` here silently becomes a
+# new empty directory instead of the config you meant to mount.
+
 services:
   otel-collector:
     image: otel/opentelemetry-collector-contrib:latest
     command: ["--config=/etc/otel-collector-config.yaml"]
     volumes:
-      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+      - ./grafana/otel-collector-config.yaml:/etc/otel-collector-config.yaml
     ports:
       - "4317:4317" # OTLP gRPC
       - "4318:4318" # OTLP HTTP
@@ -14,10 +19,20 @@ services:
 
   prometheus:
     image: prom/prometheus:latest
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      # High-resolution mergeable histograms. A p99 off classic buckets is a
+      # bucket interpolation, which is not good enough to publish.
+      - --enable-feature=native-histograms
+      # A 24h run, with room to re-query it for a while afterwards.
+      - --storage.tsdb.retention.time=30d
     volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./grafana/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus-data:/prometheus
     ports:
-      - "9090:9090"
+      # 9091, not 9090: the loadtest app owns 9090.
+      - "9091:9090"
 
   grafana:
     image: grafana/grafana:latest
@@ -30,9 +45,11 @@ services:
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
       - GF_AUTH_DISABLE_LOGIN_FORM=true
     volumes:
-      - ./provisioning/datasources:/etc/grafana/provisioning/datasources
-      - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
-      - ./dashboards:/etc/grafana/dashboards
+      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
+      - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./grafana/dashboards:/etc/grafana/dashboards
+      - grafana-data:/var/lib/grafana
 
 volumes:
   grafana-data:
+  prometheus-data:

build/dev/grafana/prometheus.yml

@@ -1,5 +1,16 @@
 global:
   scrape_interval: 10s
+  # Native histograms are only exposed over the protobuf exposition format.
+  # Without PrometheusProto first, the scrape silently falls back to classic
+  # buckets and every published percentile becomes a bucket interpolation.
+  scrape_protocols:
+    - PrometheusProto
+    - OpenMetricsText1.0.0
+    - PrometheusText0.0.4
+  # Protobuf alone is not enough: the scrape must also be told to keep the
+  # native buckets. Without this the histograms arrive and are stored as
+  # classic buckets, and every native-histogram query returns empty.
+  scrape_native_histograms: true
 
 scrape_configs:
   - job_name: "otel-collector-metrics"
@@ -8,3 +19,11 @@ scrape_configs:
   - job_name: "otel-collector-telemetry"
     static_configs:
       - targets: ["otel-collector:8888"]
+
+  # The loadtest app is scraped directly rather than routed through the
+  # collector: one hop fewer, and its native histograms arrive intact instead
+  # of round-tripping through OTLP. Shows as down when no benchmark is running.
+  - job_name: "loadtest"
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["loadtest:9090"]

build/dev/loadtest/Dockerfile

@@ -0,0 +1,5 @@
+FROM golang:1.26-alpine
+RUN go install github.com/air-verse/air@v1.61.1
+WORKDIR /app
+COPY loadtest/app/ .
+ENTRYPOINT ["air", "--build.cmd", "go build -o bin/loadtest ./cmd/loadtest", "--build.bin", "./bin/loadtest", "--build.include_ext", "go"]

build/dev/loadtest/compose.yml

@@ -0,0 +1,34 @@
+name: "outpost"
+
+services:
+  loadtest:
+    image: outpost-loadtest
+    build:
+      context: ../../../
+      dockerfile: ./build/dev/loadtest/Dockerfile
+    volumes:
+      - ../../../loadtest/app:/app
+      # Run exports land on the host, where the report tooling reads them.
+      # They are the archival record of a run and must outlive the container.
+      - ../../../loadtest/out:/data/runs
+      - go_mod_cache:/go/pkg
+      - go_build_cache:/root/.cache/go-build
+    ports:
+      - "9090:9090"
+    environment:
+      - PORT=9090
+      - OUTPOST_URL=http://api:3333/api/v1
+      - OUTPOST_API_KEY=${OUTPOST_API_KEY:-apikey}
+      - MOCK_URL=http://loadtest:9090
+      - EXPORT_DIR=/data/runs
+
+volumes:
+  go_mod_cache:
+    driver: local
+  go_build_cache:
+    driver: local
+
+# No network override: the project name above puts this on the same
+# outpost_default network as the rest of the stack. Pointing it at a separate
+# external `outpost` network left the app unable to resolve `api`, and
+# Prometheus unable to resolve `loadtest`.

contributing/loadtest/overview.md

@@ -23,7 +23,7 @@ Queries the mock webhook to confirm delivery and measure latency metrics:
 ### Start Supporting Services
 
 ```bash
-cd loadtest
+cd loadtest/k6
 docker-compose up -d
 ```
 
@@ -33,7 +33,7 @@ This starts:
 
 ### Configure Environment
 
-Use `loadtest/config/environments/local.json` or create a new one (e.g., `staging.json`):
+Use `loadtest/k6/config/environments/local.json` or create a new one (e.g., `staging.json`):
 
 ```json
 {
@@ -62,7 +62,7 @@ The mock webhook must be reachable by Outpost for event delivery to succeed.
 
 Use the default `basic.json` scenario, edit it locally, or create a new one.
 
-Default scenario at `loadtest/config/scenarios/events-throughput/basic.json`:
+Default scenario at `loadtest/k6/config/scenarios/events-throughput/basic.json`:
 
 ```json
 {

loadtest/.gitignore

@@ -1 +1,12 @@
-node_modules
+# Run exports and rendered figures are outputs of `make bench`, reproducible
+# from the spec plus a Prometheus with the run still in retention.
+out/
+report/charts/
+
+# Specs are client-side input — bench.sh reads one locally and POSTs it to the
+# app, so neither the deployment nor the repo needs a copy. The run export
+# embeds the spec that produced it, which is the record that matters.
+# example.yaml stays: it documents the format and is what `make bench` defaults
+# to on a fresh checkout.
+app/runs/*.yaml
+!app/runs/example.yaml