diff --git a/.github/pr-leakage-banned-tokens.yaml b/.github/pr-leakage-banned-tokens.yaml new file mode 100644 index 0000000..1a97901 --- /dev/null +++ b/.github/pr-leakage-banned-tokens.yaml @@ -0,0 +1,78 @@ +# pr-leakage banned tokens +# +# Parsed by .github/scripts/pr_leakage_scan.py. The parser supports a small +# subset of YAML: top-level keys `always_on:` and `context_sensitive:`, each a +# list of mappings whose keys are `id`, `pattern`, `description`, and +# optionally `adjacent_any` (inline list), `window` (int), and +# `require_quote_or_error` (bool). No anchors, no flow mappings, no block +# scalars — keep entries to one regex per line. +# +# Rule IDs are stable. Removing a rule should be a deliberate PR with a +# regression fixture explaining why. + +always_on: + - id: R1 + pattern: '\b[a-z0-9-]+\.conductor\.one\b' + description: 'tenant subdomain on conductor.one' + - id: R2 + pattern: '\b[a-z0-9-]+\.ductone\.com\b' + description: 'tenant subdomain on ductone.com (btipling.d2.ductone.com exception applied)' + - id: R3 + pattern: '\b(prod|staging|preprod)-(usw2|use1|euw1|euc1|usw1|use2)\b' + description: 'internal region/env tag from profile labels' + - id: R4 + pattern: '\bbe-(temporal-sync|temporal-worker|connector-runtime|api|sync-worker)\b' + description: 'internal service name' + - id: R5 + pattern: 'https?://app\.datadoghq\.com/' + description: 'Datadog dashboard URL' + - id: R6 + pattern: 'https?://[a-z0-9-]+\.datadoghq\.com/' + description: 'Datadog tenant URL' + - id: R7 + pattern: 'https?://linear\.app/' + description: 'Linear ticket URL' + - id: R8 + pattern: 'https?://[a-z0-9-]+\.slack\.com/' + description: 'Slack URL' + - id: R9 + pattern: 'https?://[a-z0-9-]+\.notion\.so/' + description: 'Notion URL' + - id: R10a + pattern: 'https?://docs\.google\.com/' + description: 'Google Docs URL' + - id: R10b + pattern: 'https?://drive\.google\.com/' + description: 'Google Drive URL' + - id: R11 + pattern: '\b[A-Za-z0-9._%+-]+@(conductorone\.com|ductone\.com)\b' + description: 'internal email address' + - id: R12a + pattern: '<@U[A-Z0-9]{8,}>' + description: 'Slack mention syntax' + - id: R12b + pattern: '<#C[A-Z0-9]{8,}>' + description: 'Slack channel reference syntax' + +context_sensitive: + - id: C1 + pattern: '\b[0-9A-Za-z]{20,40}\b' + description: 'C1 tenant / app / org ID in tenant-adjacent prose' + adjacent_any: [tenant, Tenant, TENANT, tnt_, app_, org_] + window: 40 + - id: C2 + pattern: '\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b' + description: 'UUID inside quoted string or near the word error (verbatim production error shape)' + require_quote_or_error: true + window: 40 + - id: C3 + pattern: '\b\d{2,4}K?\s+(users|groups|members|grants|entitlements|principals|resources)\b' + description: 'customer-sized count measurement' + - id: C4 + pattern: '\b\d{2,4}\s*(GB|TB)\s*(c1z|sync|tenant|database|sqlite)\b' + description: 'customer-sized storage measurement' + - id: C5 + pattern: '\b\d{2,3}\s*(minute|minutes|min)\b' + description: 'customer wall-clock measurement near sync/tenant/temporal context' + adjacent_any: [sync, expansion, tenant, temporal, c1z] + window: 60 diff --git a/.github/pr-leakage-customer-names.txt b/.github/pr-leakage-customer-names.txt new file mode 100644 index 0000000..a2f183a --- /dev/null +++ b/.github/pr-leakage-customer-names.txt @@ -0,0 +1,13 @@ +# pr-leakage customer-name denylist. +# +# One customer name per line. Whole-word, case-insensitive, multi-word names +# treated as `\s+` between tokens. Empty lines and lines beginning with `#` +# ignored. Adding a name is a one-file PR to this repo; the next workflow run +# in any consumer repo picks the change up because the caller stub pins @main. +# +# Seed entries are taken from the captured leak fixtures +# (.github/pr-leakage-fixtures/leaky/). Add new entries only with a fixture +# that proves the regression. + +Eli Lilly +Synthetic-Acme diff --git a/.github/pr-leakage-skip-allowlist.txt b/.github/pr-leakage-skip-allowlist.txt new file mode 100644 index 0000000..62873de --- /dev/null +++ b/.github/pr-leakage-skip-allowlist.txt @@ -0,0 +1,15 @@ +# pr-leakage skip-token allowlist. +# +# Authors listed here may bypass the scanner by including the literal token +# [skip-leakage-check] anywhere in the PR body. The token without an +# allowlisted actor is a hard fail — using the token without permission is +# strictly worse than not using it. +# +# One GitHub login per line. Empty lines and lines beginning with `#` ignored. +# +# Adding a login is a PR to this repo; the PR description documents why and +# names the Security reviewer who signed off. +# +# Initial allowlist is intentionally empty. The default path is to rewrite the +# PR body so it does not leak; the escape hatch exists for incidents where +# naming a customer in a public artifact has explicit Security sign-off. diff --git a/.github/scripts/pr_leakage_scan.py b/.github/scripts/pr_leakage_scan.py new file mode 100644 index 0000000..9d41312 --- /dev/null +++ b/.github/scripts/pr_leakage_scan.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""PR leakage scanner. + +Reads a single text input (the concatenated PR title + body + every commit +message) and applies a set of always-on and context-sensitive regexes plus an +external customer-name denylist. Exits 0 when clean, non-zero on any finding. + +Pure stdlib: no PyYAML. The banned-tokens file is parsed as a small subset of +YAML (flat keys, list items prefixed with `-`, simple `key: value` mappings, +block scalars via `|`). The format is intentionally restricted so that a single +regex per pattern is unambiguous. + +Usage: + pr_leakage_scan.py --tokens --input + [--customer-names ] + [--allowlist ] + [--actor ] + [--expect-fail] + +The --expect-fail flag inverts the exit semantics: 0 when the scanner found +≥1 finding (regression test on captured-leak fixtures), non-zero otherwise. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable + + +# btipling.d2.ductone.com is Bjorn's personal dev box and is the documented +# exception to the *.ductone.com tenant subdomain rule. +DUCTONE_EXCEPTION = "btipling.d2.ductone.com" +SKIP_TOKEN = "[skip-leakage-check]" + + +@dataclass +class Rule: + rid: str + pattern: re.Pattern + description: str + # For context-sensitive rules: list of literal strings that must appear + # within `window` chars of the match. + adjacent_any: list[str] = field(default_factory=list) + window: int = 0 + # For C2 quote-or-error context: require either a quoted enclosure or + # the word "error" within `window` chars. + require_quote_or_error: bool = False + + +@dataclass +class Finding: + rid: str + description: str + match_text: str + line_no: int + line_excerpt: str + + +def parse_tokens_file(path: Path) -> tuple[list[Rule], list[Rule]]: + """Parse the restricted YAML token file. + + Expected shape: + + always_on: + - id: R1 + pattern: '...' + description: '...' + - ... + context_sensitive: + - id: C1 + pattern: '...' + description: '...' + adjacent_any: [tenant, Tenant, tnt_] + window: 40 + - id: C2 + pattern: '...' + description: '...' + require_quote_or_error: true + window: 40 + """ + text = path.read_text() + always_on: list[Rule] = [] + context_sensitive: list[Rule] = [] + current_section: list[Rule] | None = None + current: dict | None = None + + def flush(): + nonlocal current, current_section + if current is None or current_section is None: + return + rule = Rule( + rid=current["id"], + pattern=re.compile(current["pattern"]), + description=current.get("description", ""), + adjacent_any=current.get("adjacent_any", []) or [], + window=int(current.get("window", 0) or 0), + require_quote_or_error=bool(current.get("require_quote_or_error", False)), + ) + current_section.append(rule) + current = None + + for raw in text.splitlines(): + line = raw.rstrip() + if not line or line.lstrip().startswith("#"): + continue + if line == "always_on:": + flush() + current_section = always_on + continue + if line == "context_sensitive:": + flush() + current_section = context_sensitive + continue + if line.startswith(" - "): + flush() + current = {} + kv = line[4:].strip() + if ":" in kv: + k, v = kv.split(":", 1) + current[k.strip()] = _coerce_scalar(v.strip()) + continue + if line.startswith(" ") and current is not None: + kv = line.strip() + if ":" in kv: + k, v = kv.split(":", 1) + v = v.strip() + if v.startswith("[") and v.endswith("]"): + inner = v[1:-1].strip() + items = ( + [s.strip().strip("'").strip('"') for s in inner.split(",") if s.strip()] + if inner + else [] + ) + current[k.strip()] = items + else: + current[k.strip()] = _coerce_scalar(v) + continue + flush() + return always_on, context_sensitive + + +def _coerce_scalar(v: str): + if v == "": + return "" + if (v.startswith("'") and v.endswith("'")) or (v.startswith('"') and v.endswith('"')): + return v[1:-1] + if v.lower() == "true": + return True + if v.lower() == "false": + return False + if v.isdigit(): + return int(v) + return v + + +def load_customer_names(path: Path | None) -> list[re.Pattern]: + if path is None or not path.exists(): + return [] + patterns: list[re.Pattern] = [] + for raw in path.read_text().splitlines(): + name = raw.strip() + if not name or name.startswith("#"): + continue + # Whole-word, case-insensitive. Escape so a customer name with + # punctuation does not get interpreted as a regex. + escaped = re.escape(name) + # Replace each escaped whitespace with \s+ so multi-word names match + # across runs of whitespace (newlines included). + escaped = re.sub(r"\\\s+", r"\\s+", escaped) + patterns.append(re.compile(rf"\b{escaped}\b", re.IGNORECASE)) + return patterns + + +def load_skip_allowlist(path: Path | None) -> set[str]: + if path is None or not path.exists(): + return set() + out: set[str] = set() + for raw in path.read_text().splitlines(): + login = raw.strip() + if not login or login.startswith("#"): + continue + out.add(login.lower()) + return out + + +def _line_lookup(text: str) -> list[tuple[int, int, str]]: + """Return a list of (start, end, line_text) per line for fast line lookup.""" + spans: list[tuple[int, int, str]] = [] + pos = 0 + for line in text.split("\n"): + spans.append((pos, pos + len(line), line)) + pos += len(line) + 1 + return spans + + +def _line_no_for_offset(spans: list[tuple[int, int, str]], offset: int) -> tuple[int, str]: + for i, (start, end, line) in enumerate(spans): + if start <= offset <= end: + return i + 1, line + return -1, "" + + +def _adjacent_context_ok(text: str, m: re.Match, rule: Rule) -> bool: + """Return True if the adjacency rule for a context-sensitive match is satisfied.""" + start = max(0, m.start() - rule.window) + end = min(len(text), m.end() + rule.window) + window_text = text[start:end] + if rule.adjacent_any: + if not any(token in window_text for token in rule.adjacent_any): + return False + if rule.require_quote_or_error: + # Check for a quoted enclosure around the match OR the word "error" in window. + if "error" in window_text.lower(): + return True + before = text[max(0, m.start() - rule.window) : m.start()] + after = text[m.end() : min(len(text), m.end() + rule.window)] + # Look for the nearest preceding quote and following quote of same kind. + for q in ("'", '"'): + if q in before and q in after: + return True + return False + return True + + +def scan( + text: str, + always_on: Iterable[Rule], + context_sensitive: Iterable[Rule], + customer_patterns: Iterable[re.Pattern], +) -> list[Finding]: + spans = _line_lookup(text) + findings: list[Finding] = [] + + for rule in always_on: + for m in rule.pattern.finditer(text): + matched = m.group(0) + if rule.rid == "R2" and matched.lower() == DUCTONE_EXCEPTION: + continue + line_no, line = _line_no_for_offset(spans, m.start()) + findings.append( + Finding( + rid=rule.rid, + description=rule.description, + match_text=matched, + line_no=line_no, + line_excerpt=line.strip()[:200], + ) + ) + + for rule in context_sensitive: + for m in rule.pattern.finditer(text): + if not _adjacent_context_ok(text, m, rule): + continue + matched = m.group(0) + line_no, line = _line_no_for_offset(spans, m.start()) + findings.append( + Finding( + rid=rule.rid, + description=rule.description, + match_text=matched, + line_no=line_no, + line_excerpt=line.strip()[:200], + ) + ) + + for i, pat in enumerate(customer_patterns): + for m in pat.finditer(text): + line_no, line = _line_no_for_offset(spans, m.start()) + findings.append( + Finding( + rid=f"CUST{i+1}", + description="customer-name denylist match", + match_text=m.group(0), + line_no=line_no, + line_excerpt=line.strip()[:200], + ) + ) + + return findings + + +def main() -> int: + ap = argparse.ArgumentParser(description="Scan PR text for data-leak patterns.") + ap.add_argument("--tokens", required=True, type=Path) + ap.add_argument("--input", required=True, type=Path) + ap.add_argument("--customer-names", type=Path) + ap.add_argument("--allowlist", type=Path) + ap.add_argument("--actor", default="") + ap.add_argument("--expect-fail", action="store_true", + help="invert exit code for regression-test usage") + args = ap.parse_args() + + always_on, context_sensitive = parse_tokens_file(args.tokens) + customer_patterns = load_customer_names(args.customer_names) + allowlist = load_skip_allowlist(args.allowlist) + + text = args.input.read_text() + + skip_present = SKIP_TOKEN in text + actor_ok = args.actor.lower() in allowlist if args.actor else False + + if skip_present and actor_ok: + print(f"::warning::skip honored for {args.actor}") + return 0 if not args.expect_fail else 1 + + if skip_present and not actor_ok: + print(f"::error::{SKIP_TOKEN} present but actor '{args.actor}' not allowlisted") + return 1 + + findings = scan(text, always_on, context_sensitive, customer_patterns) + + if args.expect_fail: + if findings: + print(f"expected-fail OK: {len(findings)} finding(s)") + return 0 + print("::error::expected at least one finding on this fixture; got zero") + return 1 + + if not findings: + print("scan clean: no findings") + return 0 + + for f in findings: + # GitHub Actions error annotation so the failures show up in the PR UI. + print( + f"::error file={args.input.name},line={f.line_no}::" + f"[{f.rid}] {f.description} — matched '{f.match_text}' in: {f.line_excerpt}" + ) + print(f"scan failed: {len(findings)} finding(s)") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/pr-leakage-check.yaml b/.github/workflows/pr-leakage-check.yaml new file mode 100644 index 0000000..5a66063 --- /dev/null +++ b/.github/workflows/pr-leakage-check.yaml @@ -0,0 +1,58 @@ +name: pr-leakage-check + +on: + workflow_call: + inputs: + pr_number: + description: 'Pull request number to scan' + required: true + type: number + tokens_ref: + description: 'Git ref in github-workflows to fetch token files and scanner from' + required: false + default: main + type: string + +permissions: + contents: read + pull-requests: read + +jobs: + check: + # Self-test is in a sibling workflow; skip when invoked from this repo. + if: ${{ github.repository != 'ConductorOne/github-workflows' }} + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - name: Checkout github-workflows (token files + scanner) + uses: actions/checkout@v4 + with: + repository: ConductorOne/github-workflows + ref: ${{ inputs.tokens_ref }} + path: gw + persist-credentials: false + + - name: Fetch PR title, body, and commit messages + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR: ${{ inputs.pr_number }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/leak" + gh pr view "$PR" --repo "$REPO" \ + --json title,body,commits \ + -q '.title + "\n\n" + (.body // "") + "\n\n" + ((.commits // []) | map(.messageHeadline + "\n\n" + (.messageBody // "")) | join("\n\n---\n\n"))' \ + > "$RUNNER_TEMP/leak/pr-text.txt" + + - name: Run scanner + env: + ACTOR: ${{ github.actor }} + run: | + set -euo pipefail + python3 gw/.github/scripts/pr_leakage_scan.py \ + --tokens gw/.github/pr-leakage-banned-tokens.yaml \ + --customer-names gw/.github/pr-leakage-customer-names.txt \ + --allowlist gw/.github/pr-leakage-skip-allowlist.txt \ + --actor "$ACTOR" \ + --input "$RUNNER_TEMP/leak/pr-text.txt" diff --git a/.github/workflows/pr-leakage-self-test.yaml b/.github/workflows/pr-leakage-self-test.yaml new file mode 100644 index 0000000..46d13c7 --- /dev/null +++ b/.github/workflows/pr-leakage-self-test.yaml @@ -0,0 +1,61 @@ +name: pr-leakage-self-test + +on: + push: + branches: [main] + paths: + - '.github/workflows/pr-leakage-check.yaml' + - '.github/workflows/pr-leakage-self-test.yaml' + - '.github/scripts/pr_leakage_scan.py' + - '.github/pr-leakage-banned-tokens.yaml' + - '.github/pr-leakage-customer-names.txt' + - '.github/pr-leakage-skip-allowlist.txt' + - 'tests/fixtures/leakage/**' + pull_request: + paths: + - '.github/workflows/pr-leakage-check.yaml' + - '.github/workflows/pr-leakage-self-test.yaml' + - '.github/scripts/pr_leakage_scan.py' + - '.github/pr-leakage-banned-tokens.yaml' + - '.github/pr-leakage-customer-names.txt' + - '.github/pr-leakage-skip-allowlist.txt' + - 'tests/fixtures/leakage/**' + +permissions: + contents: read + +jobs: + leaky-must-fire: + runs-on: ubuntu-latest + timeout-minutes: 2 + strategy: + fail-fast: false + matrix: + fixture: [781, 863, 865] + steps: + - uses: actions/checkout@v4 + - name: Scan leaky fixture (must fire at least once) + run: | + python3 .github/scripts/pr_leakage_scan.py \ + --tokens .github/pr-leakage-banned-tokens.yaml \ + --customer-names .github/pr-leakage-customer-names.txt \ + --allowlist .github/pr-leakage-skip-allowlist.txt \ + --input tests/fixtures/leakage/${{ matrix.fixture }}.txt \ + --expect-fail + + clean-must-pass: + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - uses: actions/checkout@v4 + - name: Scan clean fixtures (must pass) + run: | + set -euo pipefail + for f in tests/fixtures/leakage/clean/*.txt; do + echo "--- scanning $f" + python3 .github/scripts/pr_leakage_scan.py \ + --tokens .github/pr-leakage-banned-tokens.yaml \ + --customer-names .github/pr-leakage-customer-names.txt \ + --allowlist .github/pr-leakage-skip-allowlist.txt \ + --input "$f" + done diff --git a/README.md b/README.md index be59303..651f1ca 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,87 @@ The `test-flow` parameter can be: - `enable-only`: Only test enabling the account - `disable-only`: Only test disabling the account +## PR Leakage Check Workflow + +A reusable workflow that scans a PR's title, body, and every commit message +for patterns that leak customer data, internal tenant identifiers, internal +service names, or internal URLs. Pilot consumer is `ConductorOne/baton-sdk`; +every `baton-*` repo can opt in with a small caller stub. + +### Files + +| File | Purpose | +| ----------------------------------------------- | ----------------------------------------------------------------------- | +| `.github/workflows/pr-leakage-check.yaml` | The reusable workflow consumed via `workflow_call`. | +| `.github/scripts/pr_leakage_scan.py` | Pure-stdlib Python scanner (always-on + context-sensitive regexes). | +| `.github/pr-leakage-banned-tokens.yaml` | Externalized rule set. Add or tune rules here. | +| `.github/pr-leakage-customer-names.txt` | Whole-word denylist of customer names known to have leaked previously. | +| `.github/pr-leakage-skip-allowlist.txt` | GitHub logins permitted to bypass via `[skip-leakage-check]` token. | +| `.github/workflows/pr-leakage-self-test.yaml` | Regression CI inside this repo: leaky fixtures must fire, clean pass. | +| `tests/fixtures/leakage/` | Captured fixtures (leaky + clean) the self-test runs against. | + +### Wiring a baton-* repo + +Add `.github/workflows/pr-leakage.yaml` to the consumer repo: + +```yaml +name: pr-leakage + +on: + pull_request: + types: [opened, edited, synchronize, reopened] + +permissions: + contents: read + pull-requests: read + +jobs: + check: + uses: ConductorOne/github-workflows/.github/workflows/pr-leakage-check.yaml@main + with: + pr_number: ${{ github.event.pull_request.number }} + secrets: inherit +``` + +The four trigger types matter: + +- `opened` — every new PR is scanned. +- `edited` — a clean PR cannot have its body edited to add a customer name + without re-scanning. +- `synchronize` — every push picks up new commit messages. +- `reopened` — a closed-then-reopened PR is re-scanned. + +The workflow runs in the base-repo context with a read-only `GITHUB_TOKEN` +and uses the scanner from `@main` of this repo, not from the PR head, so a +fork PR cannot modify the scanner that runs on it. + +### Updating the rules + +Banned tokens and customer names live in this repo. A new pattern or name is +a one-file PR here; every consumer picks the change up on its next workflow +run because the caller stub pins `@main`. + +The self-test runs whenever any of the pr-leakage files change. Adding a new +leak shape means adding a fixture under `tests/fixtures/leakage/` so the +regression check has teeth. + +### Required check enforcement + +The workflow surfacing failures is separate from making the failure block +merge. After the consumer stub lands and the check is observed working, a +repo admin adds `pr-leakage / check` to the consumer's branch-protection +required checks. The workflow does not flip branch-protection settings. + +### Escape hatch + +Including the literal token `[skip-leakage-check]` in a PR body bypasses the +scanner only if the PR actor is listed in +`.github/pr-leakage-skip-allowlist.txt`. The token without an allowlisted +actor is a hard fail — using the token without permission is strictly worse +than not using it. The default path is to rewrite the PR body so it does not +leak; the escape hatch exists for incidents where naming a customer in a +public artifact has explicit Security sign-off. + ## Development See [release-workflow.md](docs/release-workflow.md) for testing and modification guidance. diff --git a/tests/fixtures/leakage/781.txt b/tests/fixtures/leakage/781.txt new file mode 100644 index 0000000..91c61eb --- /dev/null +++ b/tests/fixtures/leakage/781.txt @@ -0,0 +1,19 @@ +### TITLE +feat(dotc1z): opt-in slim-blob grant writer with reader hydration + +### BODY +## Summary + +Strip Entitlement and Principal from grant data blobs and rebuild identity +stubs at read time. + +## Measurements (redacted fixture) + +Validation against a Synthetic-Acme target shows the c1z is smaller after +the slim writer runs; the previous run carried 489 grants of duplicated +embedded protos. + +## Reference + +Profile of be-temporal-sync prod surfaces the same hot path on every large +sync. diff --git a/tests/fixtures/leakage/863.txt b/tests/fixtures/leakage/863.txt new file mode 100644 index 0000000..5b54631 --- /dev/null +++ b/tests/fixtures/leakage/863.txt @@ -0,0 +1,22 @@ +### TITLE +perf(expand): prefetch descendant grants to eliminate per-principal queries + +### BODY +## Summary + +Replace per-principal SQL queries in `runAction` with a single bulk prefetch. + +## Measurements (redacted fixture) + +For Synthetic-Acme: 180K users, 139K groups, 72 GB c1z, a single expansion +took 38 minutes through pure-Go SQLite. The pre-fetch reduces this to a +single paginated scan. + +## Reference + +Profile evidence from be-temporal-sync prod showed CPU pinned in +`ListGrantsForEntitlement`. + +## Link + +https://app.datadoghq.com/profiling diff --git a/tests/fixtures/leakage/865.txt b/tests/fixtures/leakage/865.txt new file mode 100644 index 0000000..8d6c90a --- /dev/null +++ b/tests/fixtures/leakage/865.txt @@ -0,0 +1,18 @@ +### TITLE +perf(sync): add WithMinCheckpointInterval syncer option + +### BODY +## Summary + +Make the minimum checkpoint interval configurable so large-tenant syncs do +not spend most of their CPU in checkpoint writes. + +## Measurements (redacted fixture) + +Profile of be-temporal-sync for the Synthetic-Acme connector: 72 GB c1z, +180K users, 139K groups. Sixty percent of CPU spent in zstd compression +inside `saveC1z`. + +## Reference + +Filed in https://linear.app/example/issue/synthetic diff --git a/tests/fixtures/leakage/clean/feat-slim-grants.txt b/tests/fixtures/leakage/clean/feat-slim-grants.txt new file mode 100644 index 0000000..4fbec0c --- /dev/null +++ b/tests/fixtures/leakage/clean/feat-slim-grants.txt @@ -0,0 +1,26 @@ +### TITLE +feat(dotc1z): opt-in slim-blob grant writer with reader hydration + +### BODY +## Summary + +Add a writer mode that strips Entitlement and Principal from grant data blobs +and rebuilds identity-only stubs at read time. Saves disk bytes on tenants +with many expanded group grants. + +## Why + +memory-pressure — Triggered on large tenants where N > 100K group memberships +expand across many users; the embedded protos in each grant duplicate the +same Resource and Entitlement data many times. + +## Test plan + +- `go test ./pkg/dotc1z/...` — pass. +- Hydration is a pure function over the existing columns; covered by unit + tests for both nil and non-nil round-trips. + +## Risk + +medium — Reader behaves identically whether or not the writer ran in slim +mode; the feature is opt-in via a syncer option. diff --git a/tests/fixtures/leakage/clean/perf-batch-expand.txt b/tests/fixtures/leakage/clean/perf-batch-expand.txt new file mode 100644 index 0000000..478ab65 --- /dev/null +++ b/tests/fixtures/leakage/clean/perf-batch-expand.txt @@ -0,0 +1,27 @@ +### TITLE +perf(pkg/sync/expand): prefetch descendant grants to avoid per-principal queries + +### BODY +## Summary + +Replace O(N) per-principal `ListGrantsForEntitlement` calls in the expand +pipeline with a single bulk pre-fetch into an in-memory map. This removes the +inner SQL query from the hot path on tenants with many source grants. + +## Why + +performance — Triggered on large tenants where N > 100K source grants. The +per-call overhead of pure-Go SQLite compounds across many point lookups even +though each lookup hits a compound index. + +## Test plan + +- `go test ./pkg/sync/expand/...` — pass, including diamond, multi-level, + cycle, and mixed-directness cases. +- `go test -bench=BenchmarkExpand` — improvement over the baseline. + +## Risk + +low — Pre-fetch is per-call; no shared cache, so concurrent runActions cannot +observe each other's writes through a stale map. Diamond-graph correctness is +covered by the existing tests.