feat: enforce export pii masking and add observability/control docs

Author: brianfunk

README.md

@@ -129,7 +129,9 @@ Before deploying, ensure quality gates pass:
 - **[AGENCY_ADMIN_GUIDE.md](docs/AGENCY_ADMIN_GUIDE.md)** - Guide for government staff
 - **[PUBLIC_USER_GUIDE.md](docs/PUBLIC_USER_GUIDE.md)** - Guide for citizens
 - **[OPERATIONS_RUNBOOK.md](docs/OPERATIONS_RUNBOOK.md)** - Production operations guide
+- **[OBSERVABILITY.md](docs/OBSERVABILITY.md)** - Logging, metrics, and alerting baseline
 - **[SECURITY_AUDIT_GUIDE.md](docs/SECURITY_AUDIT_GUIDE.md)** - Security audit procedures
+- **[CONTROL_MAPPING.md](docs/CONTROL_MAPPING.md)** - Federal-ready control evidence mapping
 - **[ACCESSIBILITY_TRACKER.md](docs/ACCESSIBILITY_TRACKER.md)** - Accessibility compliance tracking
 - **[PERFORMANCE_NOTES.md](docs/PERFORMANCE_NOTES.md)** - Performance optimization guide
 

docs/API_V1.md

@@ -93,7 +93,8 @@ Request body:
   "filters": {
     "comment_statuses": ["published"],
     "date_from": "2026-01-01T00:00:00Z",
-    "date_to": "2026-01-31T23:59:59Z"
+    "date_to": "2026-01-31T23:59:59Z",
+    "include_pii": false
   }
 }
 ```
@@ -102,6 +103,10 @@ Notes:
 
 - `export_type`: `csv | zip | combined`
 - Creates job and triggers background generation function.
+- PII behavior:
+  - `include_pii` defaults to `false`.
+  - Email fields are masked unless `include_pii=true` and caller has authorized platform role.
+  - When requesting unmasked PII, include `pii_reason` in filters.
 
 ### `GET /v1/exports/{id}`
 

docs/CONTROL_MAPPING.md

@@ -0,0 +1,58 @@
+# OpenComments Federal-Ready Control Mapping (Baseline)
+
+This baseline mapping provides implementation evidence references for common NIST/FISMA-style control families.
+It is an engineering readiness artifact, not a formal authorization package.
+
+## AC - Access Control
+
+- AC-2 / AC-3 (Account and authorization enforcement)
+  - Evidence:
+    - `supabase/migrations/20260211000100_reconcile_schema_phase1.sql`
+    - `supabase/migrations/20260211000400_agency_rls_reconciliation.sql`
+    - `src/hooks/usePermissions.ts`
+    - `src/contexts/AgencyContext.tsx`
+
+## AU - Audit and Accountability
+
+- AU-2 / AU-12 (Audit event generation and retention)
+  - Evidence:
+    - `moderation_logs` table and usage in `supabase/functions/submit-comment/index.ts`
+    - export lifecycle state transitions in `exports`
+
+## SI - System and Information Integrity
+
+- SI-10 / SI-11 (Input validation and error handling)
+  - Evidence:
+    - `src/lib/validation.ts`
+    - `src/lib/validation.test.ts`
+    - `supabase/functions/submit-comment/index.ts`
+
+## SC - System and Communications Protection
+
+- SC-7 / SC-5 (Boundary protection and throttling)
+  - Evidence:
+    - `supabase/migrations/20260211000300_api_rate_limiting.sql`
+    - `supabase/functions/public-api/index.ts`
+    - `supabase/functions/submit-comment/index.ts`
+
+## MP/PL - Data Handling and Policy
+
+- PII export minimization and masking defaults
+  - Evidence:
+    - `supabase/functions/generate-export/index.ts` (email masking by default; explicit gated inclusion)
+    - `docs/API_V1.md`
+    - `docs/DATA_DICTIONARY.md`
+
+## IR/CP - Incident and Recovery Readiness
+
+- Operational triage and recovery guidance
+  - Evidence:
+    - `docs/OPERATIONS_RUNBOOK.md`
+    - `docs/OBSERVABILITY.md`
+
+## Evidence Maintenance
+
+- Update this mapping when:
+  - control-relevant migrations are added
+  - security-critical behavior changes
+  - operational controls or runbooks are revised

docs/OBSERVABILITY.md

@@ -0,0 +1,55 @@
+# OpenComments Observability Baseline
+
+This document defines baseline operational observability for the managed-cloud deployment model (Supabase + Netlify).
+
+## Objectives
+
+- Detect platform-impacting failures quickly.
+- Preserve enough context for incident triage and audit.
+- Track key user and agency workflow reliability.
+
+## Logging Baseline
+
+## Application and Edge Functions
+
+- Every API response should include `X-Request-Id`.
+- Edge function errors must log:
+  - request identifier (or generated correlation ID)
+  - endpoint/function name
+  - principal type (`anon`, `authenticated`, `service`)
+  - stable error code and message
+- Avoid logging raw secrets and full PII values.
+
+## Database and Job Flows
+
+- Export lifecycle transitions are persisted in `exports` (`pending -> processing -> completed|failed`).
+- Moderation and submission events are persisted in `moderation_logs`.
+- Rate-limit decisions are persisted in `api_rate_limits` (windowed counters).
+
+## Operational Signals
+
+- Public API health:
+  - success rate (`2xx/3xx`)
+  - error rate (`4xx/5xx`) with `429` tracked separately
+  - latency p50/p95/p99
+- Submission pipeline:
+  - comment submit success/failure counts
+  - CAPTCHA verification failure rate
+  - attachment upload warning/failure rate
+- Export pipeline:
+  - queue depth (`pending` count)
+  - export completion time
+  - export failure rate by error class
+
+## Alerting Starter Thresholds
+
+- API error rate > 5% over 10 minutes.
+- Export failure rate > 10% over 30 minutes.
+- No successful comment submissions for 30 minutes during expected traffic windows.
+- Sudden `429` spikes (possible abuse or misconfigured client integrations).
+
+## Runbook Links
+
+- Operational handling: `docs/OPERATIONS_RUNBOOK.md`
+- Security audit process: `docs/SECURITY_AUDIT_GUIDE.md`
+- API contract: `docs/API_V1.md`

docs/plan/M06-security-and-compliance-hardening.md

@@ -16,14 +16,14 @@ Implement engineering controls for a federal-ready baseline posture.
 ## Implementation checklist
 
 - [ ] Add append-only audit logging for key actions.
-- [ ] Define PII data handling and export masking rules.
+- [x] Define PII data handling and export masking rules.
 - [ ] Add abuse detection/throttling controls.
-- [ ] Produce baseline control mapping document.
+- [x] Produce baseline control mapping document.
 
 ## Acceptance criteria
 
 - [ ] Sensitive operations are traceable and auditable.
-- [ ] PII handling behavior is documented and enforced.
+- [x] PII handling behavior is documented and enforced.
 
 ## Risks/blockers
 
@@ -36,3 +36,5 @@ Implement engineering controls for a federal-ready baseline posture.
 ## Progress log (append-only)
 
 - 2026-02-11: Scope defined.
+- 2026-02-11: Hardened export behavior in `supabase/functions/generate-export/index.ts` to mask commenter email by default and only include raw PII when explicitly requested with authorized platform role.
+- 2026-02-11: Added `docs/CONTROL_MAPPING.md` as baseline NIST/FISMA-style evidence mapping artifact.

docs/plan/M08-quality-ci-cd-and-observability.md

@@ -19,7 +19,7 @@ Restore engineering quality gates and add operational observability.
 - [x] Reduce/resolve blocking TypeScript errors.
 - [x] Add initial unit/integration tests.
 - [x] Add CI pipeline for lint, typecheck, build, tests.
-- [ ] Add basic app and job observability docs.
+- [x] Add basic app and job observability docs.
 
 ## Acceptance criteria
 
@@ -42,3 +42,4 @@ Restore engineering quality gates and add operational observability.
 - 2026-02-11: Added CI workflow `.github/workflows/ci.yml` with required `lint`, `typecheck`, `test:ci`, and `build` jobs on pull requests and pushes to `main`.
 - 2026-02-11: Added `package.json` scripts `typecheck` and `test:ci` to standardize local and CI quality commands.
 - 2026-02-11: Added first unit tests in `src/lib/validation.test.ts` and confirmed `npm run test:ci` executes real tests (6 passing).
+- 2026-02-11: Added `docs/OBSERVABILITY.md` baseline for logging, metrics, alerting thresholds, and runbook linkage.

supabase/functions/generate-export/index.ts

@@ -26,6 +26,10 @@ interface ExportComment {
   comment_attachments?: any[]
 }
 
+interface ExportOptions {
+  includePii: boolean
+}
+
 serve(async (req) => {
   if (req.method === 'OPTIONS') {
     return new Response('ok', { headers: corsHeaders })
@@ -62,25 +66,27 @@ serve(async (req) => {
       p_progress_percent: 5
     })
 
-    const { agency_id, docket_id, export_type, filters_json } = exportJob
+    const { agency_id, docket_id, export_type, filters_json, created_by } = exportJob
     const filters = filters_json || {}
+    const includePii = await resolvePiiExportPermission(supabase, created_by, filters)
+    const exportOptions: ExportOptions = { includePii }
 
     let fileContent: Uint8Array
     let fileName: string
     let contentType: string
 
     if (export_type === 'csv') {
-      const result = await generateCSVExport(supabase, agency_id, docket_id, filters)
+      const result = await generateCSVExport(supabase, agency_id, docket_id, filters, exportOptions)
       fileContent = new TextEncoder().encode(result.csv)
       fileName = `comments_export_${new Date().toISOString().split('T')[0]}.csv`
       contentType = 'text/csv'
     } else if (export_type === 'zip') {
-      const result = await generateZIPExport(supabase, agency_id, docket_id, filters)
+      const result = await generateZIPExport(supabase, agency_id, docket_id, filters, exportOptions)
       fileContent = result.zipBuffer
       fileName = `attachments_export_${new Date().toISOString().split('T')[0]}.zip`
       contentType = 'application/zip'
     } else if (export_type === 'combined') {
-      const result = await generateCombinedExport(supabase, agency_id, docket_id, filters)
+      const result = await generateCombinedExport(supabase, agency_id, docket_id, filters, exportOptions)
       fileContent = result.zipBuffer
       fileName = `combined_export_${new Date().toISOString().split('T')[0]}.zip`
       contentType = 'application/zip'
@@ -173,6 +179,47 @@ function csvCell(value: unknown): string {
   return `"${str.replace(/"/g, '""')}"`
 }
 
+function maskEmail(email?: string): string {
+  if (!email) return ''
+  const trimmed = email.trim()
+  const atIndex = trimmed.indexOf('@')
+  if (atIndex <= 1) {
+    return '***'
+  }
+  return `${trimmed[0]}***${trimmed.slice(atIndex)}`
+}
+
+async function resolvePiiExportPermission(
+  supabase: any,
+  createdBy: string | null | undefined,
+  filters: any
+): Promise<boolean> {
+  if (filters?.include_pii !== true) {
+    return false
+  }
+
+  if (!filters?.pii_reason || typeof filters.pii_reason !== 'string') {
+    return false
+  }
+
+  if (!createdBy) {
+    return false
+  }
+
+  const { data, error } = await supabase
+    .from('platform_roles')
+    .select('role')
+    .eq('user_id', createdBy)
+    .in('role', ['super_owner', 'super_user'])
+    .maybeSingle()
+
+  if (error) {
+    return false
+  }
+
+  return !!data
+}
+
 async function fetchExportComments(
   supabase: any,
   agencyId: string,
@@ -234,14 +281,14 @@ async function fetchExportComments(
   return (data || []) as ExportComment[]
 }
 
-function buildCSV(comments: ExportComment[]): string {
+function buildCSV(comments: ExportComment[], options: ExportOptions): string {
   const headers = [
     'Comment ID',
     'Docket ID',
     'Docket Title',
     'Reference Code',
     'Commenter Name',
-    'Commenter Email',
+    options.includePii ? 'Commenter Email' : 'Commenter Email (Masked)',
     'Organization',
     'Comment Text',
     'Status',
@@ -261,7 +308,7 @@ function buildCSV(comments: ExportComment[]): string {
       csvCell(docket?.title || ''),
       csvCell(docket?.reference_code || ''),
       csvCell(comment.commenter_name || ''),
-      csvCell(comment.commenter_email || ''),
+      csvCell(options.includePii ? (comment.commenter_email || '') : maskEmail(comment.commenter_email)),
       csvCell(comment.commenter_organization || ''),
       csvCell(comment.content || ''),
       csvCell(comment.status || ''),
@@ -310,13 +357,25 @@ async function addAttachmentsToZip(zip: JSZip, supabase: any, comments: ExportCo
   return attachmentCount
 }
 
-async function generateCSVExport(supabase: any, agencyId: string, docketId: string | null, filters: any) {
+async function generateCSVExport(
+  supabase: any,
+  agencyId: string,
+  docketId: string | null,
+  filters: any,
+  options: ExportOptions
+) {
   const comments = await fetchExportComments(supabase, agencyId, docketId, filters)
-  const csv = buildCSV(comments)
+  const csv = buildCSV(comments, options)
   return { csv, count: comments.length }
 }
 
-async function generateZIPExport(supabase: any, agencyId: string, docketId: string | null, filters: any) {
+async function generateZIPExport(
+  supabase: any,
+  agencyId: string,
+  docketId: string | null,
+  filters: any,
+  options: ExportOptions
+) {
   const comments = await fetchExportComments(supabase, agencyId, docketId, filters)
   const zip = new JSZip()
 
@@ -329,7 +388,8 @@ async function generateZIPExport(supabase: any, agencyId: string, docketId: stri
   zip.file('manifest.json', JSON.stringify({
     generated_at: new Date().toISOString(),
     comment_count: comments.length,
-    attachment_count: attachmentCount
+    attachment_count: attachmentCount,
+    pii_included: options.includePii
   }, null, 2))
 
   const zipBuffer = await zip.generateAsync({
@@ -344,11 +404,17 @@ async function generateZIPExport(supabase: any, agencyId: string, docketId: stri
   }
 }
 
-async function generateCombinedExport(supabase: any, agencyId: string, docketId: string | null, filters: any) {
+async function generateCombinedExport(
+  supabase: any,
+  agencyId: string,
+  docketId: string | null,
+  filters: any,
+  options: ExportOptions
+) {
   const comments = await fetchExportComments(supabase, agencyId, docketId, filters)
   const zip = new JSZip()
 
-  zip.file('comments.csv', buildCSV(comments))
+  zip.file('comments.csv', buildCSV(comments, options))
 
   const includeAttachments = filters.include_attachments !== false
   let attachmentCount = 0
@@ -362,7 +428,8 @@ async function generateCombinedExport(supabase: any, agencyId: string, docketId:
     comment_count: comments.length,
     attachment_count: attachmentCount,
     includes_comments_csv: true,
-    includes_attachments: includeAttachments
+    includes_attachments: includeAttachments,
+    pii_included: options.includePii
   }, null, 2))
 
   const zipBuffer = await zip.generateAsync({