From d26fa871c34b59a2bf46487b9f6c5f4dfe4ef331 Mon Sep 17 00:00:00 2001 From: imlogang <37604784+imlogang@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:06:43 -0500 Subject: [PATCH 1/2] Add script to forecast costs from server to cloud --- forecast-costs/README.md | 115 +++++++++++++++++++++ forecast-costs/check-usage.sh | 189 ++++++++++++++++++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 forecast-costs/README.md create mode 100755 forecast-costs/check-usage.sh diff --git a/forecast-costs/README.md b/forecast-costs/README.md new file mode 100644 index 0000000..1628067 --- /dev/null +++ b/forecast-costs/README.md @@ -0,0 +1,115 @@ +# CircleCI Server compute cost forecast + +`check-usage.sh` estimates CircleCI cloud compute credits for a CircleCI Server install by reading job durations from the `conductor_production` Postgres database and projecting credits across resource classes. + +This is a **forecast / estimate**, not a billing invoice. + +## Requirements + +| Tool | Why | +|------|-----| +| **bash** | Script shell | +| **kubectl** | Discover the Postgres pod/secret and (by default) port-forward to it | +| **psql** | Run the SQL against `conductor_production` | +| Cluster access | Your kubeconfig must be able to talk to the CircleCI Server namespace | + +### Installing `psql` locally + +You need the PostgreSQL **client** (`psql`), not a full local Postgres server. + +**macOS (Homebrew):** + +```bash +brew install libpq +brew link --force libpq # puts psql on your PATH +``` + +Or install the full package if you prefer: + +```bash +brew install postgresql@14 +``` + +**Ubuntu / Debian:** + +```bash +sudo apt-get update +sudo apt-get install -y postgresql-client +``` + +Confirm it works: + +```bash +psql --version +``` + +### Kubernetes access + +- `kubectl` configured for the cluster that hosts CircleCI Server +- Permission to `get` pods/secrets in the Server namespace, and to `port-forward` to the Postgres pod (unless you pass `-H` / `-p` to an already-reachable host) + +## Usage + +```bash +./check-usage.sh -n [-d ] [-H ] [-p ] +``` + +Make the script executable once if needed: + +```bash +chmod +x check-usage.sh +``` + +### Flags + +| Flag | Required | Default | Description | +|------|----------|---------|-------------| +| `-n` | Yes | — | Kubernetes namespace for CircleCI Server | +| `-d` | No | `30` | Look-back window in days | +| `-H` | No | port-forward via kubectl | Postgres host | +| `-p` | No | `5432` (or `15432` when port-forwarding) | Postgres port | + +### Examples + +**Default:** discover Postgres in the namespace, port-forward, query last 30 days: + +```bash +./check-usage.sh -n circleci-server +``` + +**Custom look-back window:** + +```bash +./check-usage.sh -n circleci-server -d 90 +``` + +**Connect to an already-reachable Postgres** (skip port-forward): + +```bash +./check-usage.sh -n circleci-server -H 10.0.0.50 -p 5432 +``` + +When `-H` is omitted, the script: + +1. Finds the Postgres pod (`app.kubernetes.io/name=postgresql`, then `app=postgresql`) +2. Reads the password from the Postgres secret (`postgres-password` or `postgresql-password`) +3. Port-forwards `localhost:15432` → pod `:5432` +4. Runs `psql` as user `postgres` against database `conductor_production` +5. Tears down the port-forward on exit + +If the password cannot be read from the secret, `psql` will prompt for it. + +## Output + +The script prints a table with one row per estimated resource class (`small` … `2xlarge`) plus a total: + +- **estimated_minutes** / **estimated_credits** — per resource class (and summed on the total row) +- **total_jobs** — only on the total row: actual completed jobs in the look-back window (from `job_started_events` / `job_ended_events`) + +Credits use CircleCI cloud rates (e.g. medium = 12 credits/minute). Figures are estimates and may differ from actual cloud billing. + +## Notes + +- Only jobs with both a start and end event, and `ended_at > started_at`, are included. +- The script does not modify the cluster beyond a temporary port-forward (unless you point it at an external host with `-H`). +- The script filename is `check-usage.sh`. diff --git a/forecast-costs/check-usage.sh b/forecast-costs/check-usage.sh new file mode 100755 index 0000000..653cb97 --- /dev/null +++ b/forecast-costs/check-usage.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ------------------------------------------------------------------------------ +# forecast.sh — CircleCI Server compute cost estimator +# Queries conductor_production for job durations and estimates cloud credits +# using historical cloud resource class usage weights. +# +# Usage: ./forecast.sh -n [-d ] [-H ] [-p ] +# ------------------------------------------------------------------------------ + +DAYS=30 +PG_PORT=5432 +PG_USER=postgres +PG_HOST="" +NAMESPACE="" +SECRET_NAME="" +SECRET_KEY="postgres-password" + +usage() { + cat < [-d ] [-H ] [-p ] + + -n Kubernetes namespace (required) + -d Number of days to look back (default: 30) + -H Postgres host (default: auto port-forward via kubectl) + -p Postgres port (default: 5432) +USAGE + exit 1 +} + +while getopts "n:d:H:p:" opt; do + case $opt in + n) NAMESPACE="$OPTARG" ;; + d) DAYS="$OPTARG" ;; + H) PG_HOST="$OPTARG" ;; + p) PG_PORT="$OPTARG" ;; + *) usage ;; + esac +done + +[[ -z "$NAMESPACE" ]] && { echo "Error: -n is required"; usage; } + +# ------------------------------------------------------------------------------ +# Discover PG pod — try app.kubernetes.io/name first (Bitnami), then app label +# (matches the pattern used in server-scripts/upgrade-postgres-to-14) +# ------------------------------------------------------------------------------ +echo "Looking up PostgreSQL pod in namespace '$NAMESPACE'..." + +PG_POD=$(kubectl -n "$NAMESPACE" get pods \ + -l "app.kubernetes.io/name=postgresql" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + +if [[ -z "$PG_POD" ]]; then + PG_POD=$(kubectl -n "$NAMESPACE" get pods \ + -l "app=postgresql" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) +fi + +if [[ -z "$PG_POD" ]]; then + echo "Error: Could not find a PostgreSQL pod in namespace '$NAMESPACE'" + echo " Tried labels: app.kubernetes.io/name=postgresql, app=postgresql" + echo " List pods manually: kubectl -n $NAMESPACE get pods" + exit 1 +fi +echo " Pod: $PG_POD" + +# ------------------------------------------------------------------------------ +# Discover secret — try app.kubernetes.io/name label first, then fall back to +# the well-known secret name 'postgresql' (same approach as upgrade script) +# ------------------------------------------------------------------------------ +if [[ -z "$SECRET_NAME" ]]; then + SECRET_NAME=$(kubectl -n "$NAMESPACE" get secret \ + -l "app.kubernetes.io/name=postgresql" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) +fi + +if [[ -z "$SECRET_NAME" ]]; then + if kubectl -n "$NAMESPACE" get secret postgresql &>/dev/null; then + SECRET_NAME="postgresql" + fi +fi + +if [[ -z "$SECRET_NAME" ]]; then + echo "Error: Could not auto-discover PostgreSQL secret in namespace '$NAMESPACE'" + exit 1 +fi +echo " Secret: $SECRET_NAME (key: $SECRET_KEY)" + +PG_PASSWORD=$(kubectl -n "$NAMESPACE" get secret "$SECRET_NAME" \ + -o jsonpath="{.data.$SECRET_KEY}" 2>/dev/null | base64 -d || true) + +# Some installs use 'postgresql-password' as the key +if [[ -z "$PG_PASSWORD" ]]; then + PG_PASSWORD=$(kubectl -n "$NAMESPACE" get secret "$SECRET_NAME" \ + -o jsonpath='{.data.postgresql-password}' 2>/dev/null | base64 -d || true) + [[ -n "$PG_PASSWORD" ]] && SECRET_KEY="postgresql-password" +fi + +if [[ -z "$PG_PASSWORD" ]]; then + echo "WARNING: Could not read password from secret '$SECRET_NAME' — will fall back to psql prompt." +fi + +# ------------------------------------------------------------------------------ +# Port-forward if no external host provided +# ------------------------------------------------------------------------------ +if [[ -z "$PG_HOST" ]]; then + echo "Port-forwarding $PG_POD -> localhost:15432..." + kubectl -n "$NAMESPACE" port-forward "$PG_POD" 15432:5432 &>/dev/null & + PF_PID=$! + trap 'echo ""; echo "Stopping port-forward..."; kill $PF_PID 2>/dev/null || true' EXIT + sleep 2 + PG_HOST="localhost" + PG_PORT=15432 +fi + +# ------------------------------------------------------------------------------ +# SQL query +# ------------------------------------------------------------------------------ +SQL=$(cat <= NOW() - INTERVAL '$DAYS days' + AND e.ended_at > s.started_at +), +resource_classes AS ( + SELECT * FROM (VALUES + ('small', 0.189979, 6), + ('medium', 0.428673, 12), + ('large', 0.310874, 24), + ('xlarge', 0.065853, 48), + ('2xlarge', 0.004621, 96) + ) AS t(resource_class, weight, credits_per_minute) +), +breakdown AS ( + SELECT + rc.resource_class, + ROUND((js.total_minutes * rc.weight)::numeric, 2) AS estimated_minutes, + ROUND((js.total_minutes * rc.weight * rc.credits_per_minute)::numeric, 2) AS estimated_credits + FROM job_stats js + CROSS JOIN resource_classes rc +) +SELECT resource_class, total_jobs, estimated_minutes, estimated_credits +FROM ( + SELECT + resource_class, + NULL::bigint AS total_jobs, + estimated_minutes, + estimated_credits, + 0 AS sort_order + FROM breakdown + UNION ALL + SELECT + '── TOTAL ──', + js.total_jobs, + ROUND(SUM(b.estimated_minutes)::numeric, 2), + ROUND(SUM(b.estimated_credits)::numeric, 2), + 1 AS sort_order + FROM breakdown b + CROSS JOIN job_stats js + GROUP BY js.total_jobs +) t +ORDER BY sort_order, estimated_credits DESC NULLS LAST; +ENDSQL +) + +# ------------------------------------------------------------------------------ +# Run query +# ------------------------------------------------------------------------------ +echo "" +echo "CircleCI Server Compute Cost Forecast (last $DAYS days)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +PGPASSWORD="$PG_PASSWORD" psql \ + -h "$PG_HOST" \ + -p "$PG_PORT" \ + -U "$PG_USER" \ + -d conductor_production \ + --pset="border=2" \ + --pset="format=aligned" \ + -c "$SQL" + +echo "" +echo "NOTE: Resource class breakdown uses CircleCI usage weights as a proxy." +echo " Actual distribution may vary. Credits are estimates, not exact billing figures." \ No newline at end of file From d2495b21bfc3474a107885d7b64f2a44c4b2b103 Mon Sep 17 00:00:00 2001 From: Logan G <37604784+imlogang@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:21:23 -0500 Subject: [PATCH 2/2] Update forecast-costs/check-usage.sh Co-authored-by: Atul S <3430610+atulsingh0@users.noreply.github.com> --- forecast-costs/check-usage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forecast-costs/check-usage.sh b/forecast-costs/check-usage.sh index 653cb97..68a3621 100755 --- a/forecast-costs/check-usage.sh +++ b/forecast-costs/check-usage.sh @@ -120,7 +120,7 @@ fi SQL=$(cat <