diff --git a/.gitignore b/.gitignore index a5c1f758..18dca939 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,8 @@ claude.md mission-control/src/components/datasheets/Generated*.tsx .playwright-mcp/ solutions/ +datasheets/ +*.png +**/specs/ +.facet/ +.DS_Store diff --git a/canary-checker/docs/reference/1-exec.mdx b/canary-checker/docs/reference/1-exec.mdx index bee58f8f..861fa8fa 100644 --- a/canary-checker/docs/reference/1-exec.mdx +++ b/canary-checker/docs/reference/1-exec.mdx @@ -50,7 +50,19 @@ See Image Variants for a list of in ## Shell Language -Use a shebang (`#!`) line as the first line of `script` to choose a different shell. The base image includes `python`, `bash`, and `pwsh`. +Use a shebang (`#!`) line as the first line of `script` to choose a different interpreter. The base image includes `bash`, `python`, and `pwsh` (PowerShell). Canary Checker installs additional runtimes, including Bun and UV, when first used. + +### Bun + +Use `#!/usr/bin/env bun` to run JavaScript or TypeScript with [Bun](https://bun.sh). Canary Checker detects the shebang, installs Bun when necessary, and runs the script with the configured environment and connections. + +### Python with UV + +Use `#!/usr/bin/env python3` to run Python with [`uv run --quiet`](https://docs.astral.sh/uv/). UV manages the Python runtime, virtual environment, and dependencies declared with [PEP 723](https://peps.python.org/pep-0723/) inline script metadata. + +### PowerShell + +Use `#! pwsh` or `#!/usr/bin/env pwsh` to run a PowerShell script. Canary Checker installs PowerShell when necessary and runs the script from a `.ps1` file with a non-interactive execution policy. diff --git a/canary-checker/docs/reference/1-folder.mdx b/canary-checker/docs/reference/1-folder.mdx index 4f564017..187b8873 100644 --- a/canary-checker/docs/reference/1-folder.mdx +++ b/canary-checker/docs/reference/1-folder.mdx @@ -82,6 +82,35 @@ The following example checks a local folder for files matching `pg-backups-.*.zi ```yaml title="postgres-backup-check.yaml" file=/modules/canary-checker/fixtures/datasources/folder_with_filter.yaml ``` +## Templated Paths + +The `path` field supports Go template expressions, which is particularly useful for S3 and GCS +bucket checks. Instead of scanning an entire bucket prefix on every check run, you can narrow +the listing to a specific date-partitioned or environment-scoped prefix. + +```yaml title="templated-s3-path.yaml" +apiVersion: canaries.flanksource.com/v1 +kind: Canary +metadata: + name: daily-backup-check +spec: + schedule: "@every 1h" + folder: + - name: check-todays-backups + path: 's3://my-bucket/backups/{{.properties.env}}/{{ now.Format "2006/01/02" }}/' + filter: + maxAge: 24h + minCount: 1 + minSize: 10mb +``` + +:::tip Performance +Templated paths can dramatically improve S3 check performance. A fixed path like `s3://bucket/backups/` +lists **all** objects under that prefix, which can take minutes for buckets with millions of objects. +A templated path like `s3://bucket/backups/{{ now.Format "2006/01/02" }}/` narrows the `ListObjects` +call to a single date partition. +::: + ## Result Variables The following fields are available in `test`, `display`, and `transform` [expressions](../concepts/expressions). diff --git a/canary-checker/docs/reference/1-http.mdx b/canary-checker/docs/reference/1-http.mdx index 464bbf75..dde54b27 100644 --- a/canary-checker/docs/reference/1-http.mdx +++ b/canary-checker/docs/reference/1-http.mdx @@ -37,6 +37,7 @@ The HTTP check sends HTTP requests and evaluates the response. {field: "ntlm", description: "When set to true, authenticates using NTLM v1 protocol", scheme: "bool"}, {field: "ntlmv2", description: "When set to true, authenticates using NTLM v2 protocol", scheme: "bool"}, {field: "crawl", description: "Crawl configuration for following links", scheme: "[Crawl](#crawl-configuration)"}, + {field: "dependsOn", description: "Check names that must complete before this check runs. See [Request Chaining](../concepts/request-chaining)", scheme: "[]string"}, {field: "endpoint", description: "Deprecated: Use url instead", scheme: "string", deprecated: true}, ]}/> @@ -94,6 +95,19 @@ Use `awsSigV4` to sign requests with AWS Signature Version 4. ``` +### HAR Capture + +HTTP checks can record a [HAR 1.2](http://www.softwareishard.com/blog/har-12-spec/) file containing the requests and responses collected during a Canary run. + +Enable HAR capture with the `http.har` Canary property. Set `http.har.location` to choose the output directory. + +| Property | Description | Default | +| -------- | ----------- | ------- | +| `http.har` | Enable HAR capture for HTTP traffic in the Canary | `false` | +| `http.har.location` | Directory where the HAR file is written | `.` | + +HAR files use the name `__.har`. + ### Result Variables Result variables can be used in `test`, `display` and `transform` [expressions](../concepts/expressions) diff --git a/common/src/components/AuditDiagram.tsx b/common/src/components/AuditDiagram.tsx index d71b8491..1badcada 100644 --- a/common/src/components/AuditDiagram.tsx +++ b/common/src/components/AuditDiagram.tsx @@ -15,27 +15,7 @@ import { import { HiUserGroup, HiShieldCheck, HiKey } from 'react-icons/hi2'; import { FaHistory } from 'react-icons/fa'; import BoxNode from './diagrams/BoxNode'; - -const COLORS = { - primary: '#2d7de4', - background: '#f7fbfe', - accent: '#1069dc', - muted: '#62758a', -}; - -const primaryArrowProps = { - color: COLORS.primary, - strokeWidth: 3, - headSize: 4, - dashness: { strokeLen: 10, nonStrokeLen: 5, animation: 1 }, -} as const; - -const secondaryArrowProps = { - color: COLORS.muted, - strokeWidth: 2, - headSize: 3, - dashness: { strokeLen: 6, nonStrokeLen: 4, animation: 1 }, -} as const; +import { COLORS, primaryArrowProps, secondaryArrowProps } from './diagrams/diagramUtils'; interface IconGridProps { items: Array<{ Icon: React.ComponentType<{ className?: string }>; label?: string }>; diff --git a/common/src/components/CustomScraperDiagram.tsx b/common/src/components/CustomScraperDiagram.tsx new file mode 100644 index 00000000..1e1748bf --- /dev/null +++ b/common/src/components/CustomScraperDiagram.tsx @@ -0,0 +1,211 @@ +import React, { useId } from 'react'; +import BrowserOnly from '@docusaurus/BrowserOnly'; +import Xarrow from 'react-xarrows'; +import BoxNode from './diagrams/BoxNode'; +import { + ConfigDbWhite, + MissionControlWhite, + Http, + Postgres, + SqlServer, + Clickhouse, + Aws, + Azure, + GoogleCloud, + AzureAd, + Github, +} from '@flanksource/icons/mi'; +import { HiUserGroup, HiShieldCheck, HiKey } from 'react-icons/hi2'; +import { FaHistory, FaLightbulb, FaDatabase, FaFileCode, FaTerminal } from 'react-icons/fa'; +import { COLORS, primaryArrowProps } from './diagrams/diagramUtils'; + +function ProtocolBox({ id, title, icons }: { id: string; title: string; icons: Array<{ Icon: React.ComponentType<{ className?: string }>; label: string }> }) { + return ( +
+ +
+ {icons.map(({ Icon, label }) => ( +
+ + {label} +
+ ))} +
+ +
+ ); +} + +function DataSourcesColumn({ idFile, idHttp, idSql }: { idFile: string; idHttp: string; idSql: string }) { + return ( +
+ + + +
+ ); +} + +function ScraperBox({ id }: { id: string }) { + return ( +
+ +
+ ScrapeConfig + full: true +
+
+ ); +} + +function JsonOutputBox({ id }: { id: string }) { + const lines = [ + { text: '{', indent: 0 }, + { text: '"id": "db-prod-001",', indent: 1 }, + { text: '"type": "Database",', indent: 1 }, + { text: '"config": { ... },', indent: 1, highlight: true }, + { text: '"changes": [ ... ],', indent: 1, highlight: true }, + { text: '"access": [ ... ],', indent: 1, highlight: true }, + { text: '"logs": [ ... ],', indent: 1, highlight: true }, + { text: '"analysis": [ ... ],', indent: 1, highlight: true }, + { text: '"users": [ ... ]', indent: 1, highlight: true }, + { text: '}', indent: 0 }, + ]; + + return ( +
+ + 
+          {lines.map(({ text, indent, highlight }, i) => (
+            

+              {text}
+            

+          ))}
+
+ +
+ ); +} + +function MissionControlBox({ id }: { id: string }) { + return ( +
+
+
+ + Mission Control +
+
+
+ {[ + { Icon: FaDatabase, label: 'Config Item' }, + { Icon: FaHistory, label: 'Changes' }, + { Icon: HiKey, label: 'Access Records' }, + { Icon: HiUserGroup, label: 'Access Logs' }, + { Icon: FaLightbulb, label: 'Insights' }, + { Icon: HiShieldCheck, label: 'Users & Roles' }, + ].map(({ Icon, label }) => ( +
+ + {label} +
+ ))} +
+
+ ); +} + +function TransformPipeline({ id }: { id: string }) { + const steps = ['Exclude', 'Mask', 'Relationships', 'Changes']; + return ( +
+ {steps.map((step, i) => ( + +
+ {step} +
+ {i < steps.length - 1 && →} + + ))} +
+ ); +} + +interface CustomScraperDiagramProps { + className?: string; +} + +function CustomScraperDiagramInner({ className }: CustomScraperDiagramProps) { + const prefix = useId(); + const id = (name: string) => `${prefix}-${name}`; + + return ( +
+ + +
+ + +
+ + + + + + + + + +
+ ); +} + +export default function CustomScraperDiagram(props: CustomScraperDiagramProps) { + return ( + }> + {() => } + + ); +} diff --git a/common/src/components/EntraDataFlowDiagram.tsx b/common/src/components/EntraDataFlowDiagram.tsx index 21989ef4..99cd8aa2 100644 --- a/common/src/components/EntraDataFlowDiagram.tsx +++ b/common/src/components/EntraDataFlowDiagram.tsx @@ -9,42 +9,7 @@ import { MissionControlWhite, } from '@flanksource/icons/mi'; import BoxNode from './diagrams/BoxNode'; - -const COLORS = { - primary: '#2d7de4', - background: '#f7fbfe', - accent: '#1069dc', - muted: '#62758a', - outputBorder: '#10b981', -}; - -const pillStyle: React.CSSProperties = { - color: COLORS.muted, - backgroundColor: COLORS.background, - border: `1px solid ${COLORS.primary}`, -}; - -const primaryArrowProps = { - color: COLORS.primary, - strokeWidth: 3, - headSize: 4, - dashness: { strokeLen: 10, nonStrokeLen: 5, animation: 1 }, -} as const; - -const secondaryArrowProps = { - color: COLORS.muted, - strokeWidth: 2, - headSize: 3, - dashness: { strokeLen: 6, nonStrokeLen: 4, animation: 1 }, -} as const; - -function NodePill({ children }: { children: React.ReactNode }) { - return ( -
- {children} -
- ); -} +import { COLORS, primaryArrowProps, secondaryArrowProps, NodePill } from './diagrams/diagramUtils'; function NodeSection({ title, items, id }: { title: string; items: string[]; id?: string }) { return ( diff --git a/common/src/components/MCPDiagram.tsx b/common/src/components/MCPDiagram.tsx new file mode 100644 index 00000000..66fe2ffa --- /dev/null +++ b/common/src/components/MCPDiagram.tsx @@ -0,0 +1,125 @@ +import React, { useId } from 'react'; +import BrowserOnly from '@docusaurus/BrowserOnly'; +import Xarrow from 'react-xarrows'; +import { + Mcp, + MissionControlWhite, + ConfigDbWhite, + CanaryCheckerWhite, + Playbook, +} from '@flanksource/icons/mi'; +import { PiBrain } from 'react-icons/pi'; +import { SiClaude } from 'react-icons/si'; +import { VscCode } from 'react-icons/vsc'; +import { HiCommandLine } from 'react-icons/hi2'; +import BoxNode from './diagrams/BoxNode'; +import { COLORS, primaryArrowProps } from './diagrams/diagramUtils'; + +function AIClientsBox({ id }: { id: string }) { + const clients = [ + { Icon: SiClaude, label: 'Claude Desktop' }, + { Icon: VscCode, label: 'VS Code' }, + { Icon: HiCommandLine, label: 'Claude Code' }, + ]; + + return ( +
+ +
+ {clients.map(({ Icon, label }) => ( +
+ + {label} +
+ ))} +
+ +
+ ); +} + +function MCPEndpointBox({ id }: { id: string }) { + const tools = [ + { Icon: ConfigDbWhite, label: 'Catalog' }, + { Icon: CanaryCheckerWhite, label: 'Health Checks' }, + { Icon: Playbook, label: 'Playbooks' }, + { Icon: PiBrain, label: 'Views' }, + ]; + + return ( +
+
+
+ + Mission Control +
+
+
+
+ + /mcp +
+
+ {tools.map(({ Icon, label }) => ( +
+ + {label} +
+ ))} +
+
+
+ ); +} + +interface MCPDiagramProps { + className?: string; +} + +function MCPDiagramInner({ className }: MCPDiagramProps) { + const prefix = useId(); + const id = (name: string) => `${prefix}-${name}`; + + return ( +
+ + + + + HTTP + SSE + + }} + /> +
+ ); +} + +export default function MCPDiagram(props: MCPDiagramProps) { + return ( + }> + {() => } + + ); +} diff --git a/common/src/components/ViewsDiagram.tsx b/common/src/components/ViewsDiagram.tsx new file mode 100644 index 00000000..0a1c9635 --- /dev/null +++ b/common/src/components/ViewsDiagram.tsx @@ -0,0 +1,165 @@ +import React, { useId } from 'react'; +import BrowserOnly from '@docusaurus/BrowserOnly'; +import Xarrow from 'react-xarrows'; +import { + Prometheus, + Postgres, + Http, + K8S, + ConfigDb, + Changes, + Slack, +} from '@flanksource/icons/mi'; +import { FaFilePdf, FaFileCsv, FaFileCode, FaMarkdown } from 'react-icons/fa'; +import { VscJson } from 'react-icons/vsc'; +import BoxNode from './diagrams/BoxNode'; +import { COLORS, primaryArrowProps, outputArrowProps, NodePill } from './diagrams/diagramUtils'; + +interface IconRowProps { + items: Array<{ Icon: React.ComponentType<{ className?: string }>; label: string }>; +} + +function IconRow({ items }: IconRowProps) { + return ( +
+ {items.map(({ Icon, label }) => ( +
+ + {label} +
+ ))} +
+ ); +} + +interface ViewsDiagramProps { + className?: string; +} + +function ViewsDiagramInner({ className }: ViewsDiagramProps) { + const prefix = useId(); + const id = (name: string) => `${prefix}-${name}`; + + return ( +
+ + {/* Col 1: Sources */} +
+
+ + + +
+
+ + + +
+
+ + + +
+
+ + viewTableSelector + +
+
+ + {/* Col 2: Queries */} +
+
+ +
+ $(var.key) + each → SQLite table +
+ +
+
+ + {/* Col 3: SQLite */} +
+
+ +
+ optional merge SQL + join / aggregate +
+ +
+
+ + {/* Col 4: Outputs */} +
+
+ + pie · gauge · number + +
+
+ + typed columns + +
+
+ +
+ {[ + { Icon: FaFilePdf, label: 'PDF' }, + { Icon: FaFileCsv, label: 'CSV' }, + { Icon: VscJson, label: 'JSON' }, + { Icon: FaFileCode, label: 'HTML' }, + { Icon: FaMarkdown, label: 'MD' }, + { Icon: Slack, label: 'Slack' }, + ].map(({ Icon, label }) => ( +
+ + {label} +
+ ))} +
+ +
+
+ + {/* Arrows: Sources → Queries (fan-in) */} + + + + + + {/* Arrow: Queries → SQLite */} + + + {/* Arrows: SQLite → Outputs (fan-out) */} + + + +
+ ); +} + +export default function ViewsDiagram(props: ViewsDiagramProps) { + return ( + }> + {() => } + + ); +} diff --git a/common/src/components/ViewsVariablesDiagram.tsx b/common/src/components/ViewsVariablesDiagram.tsx new file mode 100644 index 00000000..43686dd0 --- /dev/null +++ b/common/src/components/ViewsVariablesDiagram.tsx @@ -0,0 +1,180 @@ +import React, { useId } from 'react'; +import BrowserOnly from '@docusaurus/BrowserOnly'; +import Xarrow from 'react-xarrows'; +import { + Prometheus, + Postgres, + Http, + ConfigDb, + Changes, + Slack, +} from '@flanksource/icons/mi'; +import { FaFilePdf, FaFileCsv, FaFileCode, FaMarkdown } from 'react-icons/fa'; +import { VscJson } from 'react-icons/vsc'; +import BoxNode from './diagrams/BoxNode'; +import { COLORS, primaryArrowProps, outputArrowProps, NodePill } from './diagrams/diagramUtils'; + +interface IconGridProps { + items: Array<{ Icon: React.ComponentType<{ className?: string }>; label: string }>; +} + +function IconGrid({ items }: IconGridProps) { + return ( +
+ {items.map(({ Icon, label }) => ( +
+ + {label} +
+ ))} +
+ ); +} + +interface ViewsVariablesDiagramProps { + className?: string; +} + +function ViewsVariablesDiagramInner({ className }: ViewsVariablesDiagramProps) { + const prefix = useId(); + const id = (name: string) => `${prefix}-${name}`; + + return ( +
+ + {/* Col 1: Data Sources */} +
+
+ + + +
+
+ + {/* Col 2: Named Queries */} +
+
+ + each → SQLite table + +
+
+ + {/* Col 3: In-memory SQLite */} +
+
+ + optional merge SQL + +
+
+ + {/* Col 4: PostgreSQL */} +
+
+ + + view_<ns>_<name> + + +
+
+ + {/* Col 5: PostgREST API */} +
+
+ +
+ Column Filters + → WHERE clauses +
+ +
+
+ + {/* Col 6: Outputs */} +
+
+ + visualizations + +
+
+ + typed columns + +
+
+ +
+ {[ + { Icon: FaFilePdf, label: 'PDF' }, + { Icon: FaFileCsv, label: 'CSV' }, + { Icon: VscJson, label: 'JSON' }, + { Icon: FaFileCode, label: 'HTML' }, + { Icon: FaMarkdown, label: 'MD' }, + { Icon: Slack, label: 'Slack' }, + ].map(({ Icon, label }) => ( +
+ + {label} +
+ ))} +
+ +
+
+ + {/* Arrow: Sources → Queries with $(var.key) label */} + + $(var.key) + + ), + }} + /> + + {/* Arrows: Queries → SQLite → PostgreSQL → API */} + + + + + {/* Arrows: API → Outputs (fan-out) */} + + + +
+ ); +} + +export default function ViewsVariablesDiagram(props: ViewsVariablesDiagramProps) { + return ( + }> + {() => } + + ); +} diff --git a/common/src/components/diagrams/diagramUtils.tsx b/common/src/components/diagrams/diagramUtils.tsx new file mode 100644 index 00000000..3ef11629 --- /dev/null +++ b/common/src/components/diagrams/diagramUtils.tsx @@ -0,0 +1,45 @@ +import React from 'react'; + +export const COLORS = { + primary: '#2d7de4', + outputBorder: '#10b981', + background: '#f7fbfe', + accent: '#1069dc', + muted: '#62758a', + sqliteBg: '#dae8fc', +}; + +export const primaryArrowProps = { + color: COLORS.primary, + strokeWidth: 3, + headSize: 4, + dashness: { strokeLen: 10, nonStrokeLen: 5, animation: 1 }, +} as const; + +export const secondaryArrowProps = { + color: COLORS.muted, + strokeWidth: 2, + headSize: 3, + dashness: { strokeLen: 6, nonStrokeLen: 4, animation: 1 }, +} as const; + +export const outputArrowProps = { + color: COLORS.outputBorder, + strokeWidth: 2, + headSize: 4, + dashness: { strokeLen: 8, nonStrokeLen: 4, animation: 1 }, +} as const; + +const pillStyle: React.CSSProperties = { + color: COLORS.muted, + backgroundColor: COLORS.background, + border: `1px solid ${COLORS.primary}`, +}; + +export function NodePill({ children }: { children: React.ReactNode }) { + return ( +
+ {children} +
+ ); +} diff --git a/docs/.DS_Store b/docs/.DS_Store deleted file mode 100644 index 339e27ef..00000000 Binary files a/docs/.DS_Store and /dev/null differ diff --git a/docs/specs/REQUIREMENTS-audit-report-guide.md b/docs/specs/REQUIREMENTS-audit-report-guide.md deleted file mode 100644 index a2084a78..00000000 --- a/docs/specs/REQUIREMENTS-audit-report-guide.md +++ /dev/null @@ -1,72 +0,0 @@ -# Feature: Audit Report Guide - -## Overview - -A comprehensive guide that shows platform engineers and compliance officers how to use Mission Control to build audit-ready infrastructure reporting. Covers application/config access tracking, backup monitoring, change tracking & drift detection, and infrastructure inventory. - -**Audience**: Platform engineers setting up Mission Control for audit readiness AND compliance officers reviewing audit data. - -**Location**: Top-level guide at `guide/audit/` since it spans config-db scrapers, health checks, views, and notifications. - -## Functional Requirements - -### FR-1: Application & Config Access Tracking -**Description**: Document how to scrape and report on who has access to which applications, configs, IAM roles, and group memberships. -**Acceptance Criteria**: -- [ ] Explains access log scraping with `full: true` -- [ ] Shows AWS IAM access scraper example -- [ ] Shows Kubernetes RBAC scraper example -- [ ] Shows PostgreSQL user/role scraper example -- [ ] Includes view YAML for access summary dashboard -- [ ] Links to existing access-logs and config_access reference docs - -### FR-2: Backup Monitoring -**Description**: Document how to monitor backup health, track backup locations/regions, verify retention, and test restores. -**Acceptance Criteria**: -- [ ] Backup health checks (canary checks verifying backups run on schedule) -- [ ] Backup location/region tracking (scraping backup configs for S3 buckets, regions, cross-region replication) -- [ ] Backup events from changes (tracking backup-related config changes) -- [ ] Backup restore testing (playbooks or checks that verify restore capability) -- [ ] View YAML for backup status dashboard - -### FR-3: Change Tracking & Drift Detection -**Description**: Document how to track config changes over time, identify who changed what, and detect drift. -**Acceptance Criteria**: -- [ ] Explains how config changes are captured (diff and event-based) -- [ ] Shows CloudTrail integration for AWS change tracking -- [ ] Shows Kubernetes event tracking -- [ ] View YAML for change audit dashboard -- [ ] Notification config for critical changes - -### FR-4: Infrastructure Inventory -**Description**: Document how to maintain a complete catalog of infrastructure resources with health status and relationships. -**Acceptance Criteria**: -- [ ] Shows AWS scraper for complete infrastructure inventory -- [ ] Shows Kubernetes scraper for cluster inventory -- [ ] Shows database scraper for database inventory -- [ ] View YAML for infrastructure inventory dashboard -- [ ] Explains health status and relationships for audit context - -### FR-5: Notifications for Audit Events -**Description**: Configure alerts for audit-relevant events. -**Acceptance Criteria**: -- [ ] Notification for failed backups -- [ ] Notification for critical config changes -- [ ] Notification for access anomalies (e.g., access without MFA) - -## Infrastructure Types Covered -- AWS (RDS, S3, EC2, IAM) -- Kubernetes (RBAC, resources, events) -- Databases (PostgreSQL users, roles, connection logs) - -## Document Structure -- Top-level guide at `guide/audit/index.mdx` -- End-to-end YAML examples inline -- Links to existing reference docs -- Framework-agnostic (not tied to specific compliance frameworks) - -## Success Criteria -- [ ] Guide covers all 4 audit areas with working YAML examples -- [ ] Both engineers and compliance officers can follow the guide -- [ ] Links correctly to existing reference documentation -- [ ] Follows existing doc format (frontmatter, MDX components, code blocks) diff --git a/docs/specs/REQUIREMENTS-entra-diagram-update.md b/docs/specs/REQUIREMENTS-entra-diagram-update.md deleted file mode 100644 index a5afd577..00000000 --- a/docs/specs/REQUIREMENTS-entra-diagram-update.md +++ /dev/null @@ -1,172 +0,0 @@ -# Feature: Entra Integration Data Flow Diagram Update - -## Overview - -Update the `EntraDataFlowDiagram.tsx` component to clearly communicate the left-to-right data flow of the Entra ID integration with Mission Control. The diagram should emphasize: - -1. **Left**: Entra ID as the data source (with sign-in logs as a sub-item) -2. **Middle**: Mission Control showing both scraper ingestion mechanisms and the stored catalog data -3. **Right**: Application CRD as the mapping layer producing visual audit reports - -## Current State - -**File**: `common/src/components/EntraDataFlowDiagram.tsx` -**Used in**: `mission-control/docs/integrations/azure-ad/index.mdx` - -Current layout has 3 columns but scrapers (HTTP, Logs, Event Hub) and User Logins are shown as separate nodes floating on the left alongside Entra ID, making the flow unclear. The Mission Control box only shows stored data types. The outputs show Application CRD and Audit Views as peers. - -## Functional Requirements - -### FR-1: Left Column — Entra ID Source - -**Description**: Entra ID box should be the single source on the left, containing all the data types that Mission Control scrapes. - -**Contents of Entra ID box**: -- Users -- Groups -- App Registrations -- Enterprise Apps -- Role Assignments -- Auth Policies -- Sign-in Logs (moved from the separate "User Logins" node) - -**Acceptance Criteria**: -- [ ] Single Entra ID box on the left -- [ ] Sign-in Logs included inside the Entra box (remove separate User Logins node) -- [ ] All 7 data types displayed in the box - -### FR-2: Middle Column — Mission Control (Scrapers + Catalog) - -**Description**: The Mission Control box should show two logical layers: the scraper mechanisms that ingest data, and the config items that are stored in the catalog. - -**Structure**: -- **Ingestion layer** (top or left side of MC box): HTTP Scraper, Logs Scraper, Event Hub — these are the mechanisms that pull data from Entra -- **Catalog layer** (main body of MC box): Users & Groups, Access Records, Secrets & Certs, Auth Policies — these are the config items stored in the catalog - -**Acceptance Criteria**: -- [ ] Mission Control box shows scrapers as an ingestion boundary -- [ ] Catalog data types shown as stored items inside MC -- [ ] Visual distinction between scraper mechanisms and stored data -- [ ] Arrows from Entra connect to the scraper mechanisms - -### FR-3: Right Column — Application CRD + Audit Views - -**Description**: The right side shows how Applications map catalog data into visual audit reports. - -**Application CRD sub-items**: -- Access Control -- Backups -- Cost -- Environments - -**Audit Views** (rendered outputs): -- Who Accessed, When -- Expired Secrets -- (or similar concrete report names) - -**Acceptance Criteria**: -- [ ] Application CRD shown as the mapping layer with sub-items -- [ ] Audit Views shown as the rendered output -- [ ] Clear visual flow: MC catalog → Application CRD → Audit Views (or MC → both) - -### FR-4: Left-to-Right Flow - -**Description**: The entire diagram must read clearly left to right: Source → Ingestion → Storage → Mapping → Reports. - -**Acceptance Criteria**: -- [ ] Data flows strictly left to right -- [ ] Arrows use Xarrow with existing animation style -- [ ] No vertical-only flow sections that break the L→R reading pattern - -## Technical Considerations - -- Reuse existing `BoxNode` component from `./diagrams/BoxNode` -- Reuse existing color scheme (`COLORS` object) -- Keep `BrowserOnly` wrapper for SSR compatibility -- Use `react-xarrows` for connectors -- Icons from `@flanksource/icons/mi` and `react-icons` -- Component must remain a default export for the existing import in `azure-ad/index.mdx` - -## Visual Layout (ASCII) - -``` -┌─────────────┐ ┌──────────────────────────────────┐ ┌─────────────────┐ -│ Entra ID │ │ Mission Control │ │ Application CRD │ -│ │ │ ┌────────────┐ ┌─────────────┐ │ │ │ -│ Users │────▶│ │HTTP Scraper│ │Users & Groups│ │────▶│ Access Control │ -│ Groups │ │ │Logs Scraper│ │Access Records│ │ │ Backups │ -│ App Regs │ │ │Event Hub │ │Secrets/Certs │ │ │ Cost │ -│ Ent. Apps │ │ └────────────┘ │Auth Policies │ │ │ Environments │ -│ Role Assign │ │ (ingestion) └─────────────┘ │ └─────────────────┘ -│ Auth Policy │ │ (catalog) │ │ -│ Sign-in Logs│ └──────────────────────────────────┘ ▼ -└─────────────┘ ┌─────────────────┐ - │ Audit Views │ - │ Who Accessed │ - │ Expired Secrets │ - └─────────────────┘ -``` - -## Review Feedback - -### Gemini (UX/Information Architecture) - -**Positives**: -- Moving scrapers into MC box correctly shows MC as the active agent -- Folding User Logins into Entra reduces visual noise -- Left-to-right data lifecycle layout is a strong improvement - -**Recommendations**: -1. **Group Entra items** into Identity Objects (Users, Groups, Apps, etc.) and Activity Logs (Sign-in Logs) for cleaner arrow routing -2. **Use subtle background color difference** between Ingestion and Catalog sub-sections in MC box -3. **Differentiate CRD vs Views** visually — CRD is control/config (dashed border?), Views are output (brighter border?) -4. **Avoid spaghetti arrows** — use one thick main pipe from Entra to MC, or fan out only at scrapers -5. **Set `min-width`** on container (e.g. `min-w-[1000px]`) for mobile — allow horizontal scroll -6. **Consider internal MC flow** — visual indication that Ingestion feeds Catalog (subtle internal arrow or left-to-right placement) - -### Codex (Code Quality) - -**High priority**: -- Static DOM ids (`entra-*`) cause collisions if diagram renders twice — use `useId`-prefixed ids or refs - -**Medium priority**: -- Hard-coded Xarrow offsets (lines 169-183) will break after layout change — use anchor elements inside MC box sections instead -- Repeated pill markup in 4 places — extract `NodePill` / `NodeSection` components driven by arrays - -**Recommended refactor pattern**: -```tsx -const MC_SECTIONS = [ - { id: 'ingestion', title: 'Ingestion', items: ['HTTP Scraper', 'Logs Scraper', 'Event Hub'] }, - { id: 'catalog', title: 'Catalog', items: ['Users & Groups', 'Access Records', 'Secrets & Certs', 'Auth Policies'] }, -]; -``` - -- Extract shared `primaryArrowProps` and `secondaryArrowProps` to reduce Xarrow duplication -- Move repeated inline styles into a `pillStyle` object - -## Implementation Checklist - -### Phase 1: Refactor Component Structure -- [ ] Extract `NodePill` component for repeated pill markup -- [ ] Extract `NodeSection` component for section groupings -- [ ] Use `useId`-prefixed ids instead of static ids -- [ ] Create shared arrow prop objects (`primaryArrowProps`, `secondaryArrowProps`) - -### Phase 2: Update Layout -- [ ] Group Entra items into Identity Objects + Activity Logs sections -- [ ] Move Sign-in Logs into Entra box, remove separate User Logins node -- [ ] Restructure MC box with Ingestion (left) + Catalog (right) sub-sections -- [ ] Add anchor elements inside MC sections for Xarrow targeting -- [ ] Update Application CRD sub-items (Access Control, Backups, Cost, Environments) -- [ ] Differentiate CRD (control) vs Views (output) styling - -### Phase 3: Arrows & Wiring -- [ ] Single main arrow from Entra to MC ingestion section -- [ ] Internal arrows from Ingestion anchors to Catalog (or implicit left-to-right placement) -- [ ] MC Catalog to Application CRD arrow -- [ ] Application CRD to Audit Views arrow (or MC to both) -- [ ] Set `min-width` on container for mobile - -### Phase 4: Verify & Commit -- [ ] Verify in browser with `npm run dev` -- [ ] Stage and commit changes diff --git a/mission-control-chart b/mission-control-chart index 56c1597b..5dcaea26 160000 --- a/mission-control-chart +++ b/mission-control-chart @@ -1 +1 @@ -Subproject commit 56c1597b7ab63405de74a5953aa5dacaa3e7b978 +Subproject commit 5dcaea266a16b37730ca18e92801b6c33ec79f0b diff --git a/mission-control/.gitignore b/mission-control/.gitignore index de0d97fb..338c4509 100644 --- a/mission-control/.gitignore +++ b/mission-control/.gitignore @@ -28,3 +28,4 @@ starlight/ .*cache branding/ .playwright-mcp/ +.todos/ diff --git a/mission-control/AGENTS.md b/mission-control/AGENTS.md index 42fd91a1..a6bdab92 100644 --- a/mission-control/AGENTS.md +++ b/mission-control/AGENTS.md @@ -832,7 +832,7 @@ import MyDiagram from '@site/../common/src/components/MyDiagram' ### Static Images -Store images alongside the `.mdx` file that uses them. Use standard markdown: +Store images alongside the `.mdx` file that uses them. Use standard Markdown: ```mdx ![Description of what the image shows](./image-name.png) diff --git a/mission-control/docs/changes.md b/mission-control/docs/changes.md new file mode 100644 index 00000000..d0198b80 --- /dev/null +++ b/mission-control/docs/changes.md @@ -0,0 +1,86 @@ +--- +title: "What's New" +hide_title: true +--- + +# What's New + +## March 2026 + +### Features +- **Playbooks**: Scheduled playbook execution support +- **Playbooks**: Scoped impersonation for playbook runs +- **API**: Facet connection type and rendering support +- **API**: Report action and scheduled playbook triggers +- **API**: CRD status conditions for Application, Team, IncidentRule, Notification, Scope, Permission +- **API**: CRD ready-condition status lifecycle for Connection +- **API**: ObservedGenerationSetter for all CRD types +- **Config DB**: Merge external entities with overlapping aliases +- **Config DB**: User/group alias support for external_user_groups +- **Config DB**: Unified fixture framework with e2e DB test runner +- **Duty**: Config access summary by user and by config views +- **Duty**: Grouping and deduplication of log lines +- **Duty**: ClientOption pattern with HAR collector support +- **Duty**: Stored procedures for external entity merge and upsert +- **Duty**: Include config path in changes view +- **Duty**: Case insensitive resource selector search + +### Fixes +- **Kubernetes**: Use dynamic informer factory to support CRD watches +- **CloudTrail**: Add adaptive retry +- **Notifications**: Add SMTP context to shoutrrr send errors +- **Metrics**: Remove last login query against users view +- **Config DB**: Fix FK errors on external entities +- **Config DB**: Recognize `id` as alias for `external_id` in applyConfigRefDefaults +- **Duty**: Honor DisableRLS for generated view tables +- **Duty**: Use check_id instead of id in check event trigger +- **Duty**: Drop notification_update_enqueue trigger +- **Duty**: Add event_id to event_queue index +- **Duty**: Remove custom JSON marshaler from HTTPConnection + +### Improvements +- **Views**: Add config_access_summary views + +## February 2026 + +### Features +- **Playbooks**: Scrub secret params from action output +- **Playbooks**: Template HTTP method and headers in actions +- **Connections**: Add Elasticsearch/Redis CRD types, align AWS/GCP/HTTP with duty +- **RBAC**: Sync playbook permissions to config_access table +- **Config DB**: GitHub/OpenSSF scrapers +- **Config DB**: HTTP pagination support for scraper +- **Config DB**: Create config access logs for IAM assume role +- **Config DB**: Scrape playbooks, people, teams and playbook roles +- **Config DB**: RBAC config access +- **Duty**: Shell sandbox with flanksource/sandbox-runtime +- **Duty**: AWS SigV4 signing and Azure/AWS HTTP connection types +- **Duty**: HTTP query support in dataquery package +- **Duty**: Add user type in config access summary +- **Duty**: Add source column to config_access table +- **Duty**: Kubernetes apiVersion/Kind format in QueryResources +- **Canary Checker**: Support templating path for folder check +- **Canary Checker**: Handle Secret Lookup Rate Limit errors +- **Canary Checker**: Add apiVersion field to KubernetesCheck +- **Canary Checker**: ARM matrix build with container self-tests + +### Fixes +- **Playbooks**: Keep playbook notification body authoritative +- **Playbooks**: Add connection permission checks to log actions +- **Playbooks**: Record CRD validation failures in status +- **Playbooks**: Notification connection should be templatable +- **Application**: Deduplicate users list and populate missing fields +- **Application**: Skip scraper generation for non-azure scrapers +- **Config DB**: Restore kubernetes scraper tags lost in refactor +- **Duty**: Split suppressed count into suppressed and in_progress +- **Duty**: Allow empty connection type in NewHTTPConnection +- **Duty**: Preserve inline fields during connection hydration +- **Duty**: Add missing FK indexes and constraints +- **Duty**: Skip duty auth wrapper for tokenFile auth +- **Duty**: Send POST/PUT/PATCH body in HTTP dataquery +- **Duty**: Include Go comments in generated schemas +- **Canary Checker**: Improve error output in canary-checker run + +### Improvements +- **Duty**: Connection merge headers instead of replacing on override +- **Duty**: Ordered playbook actions by scheduled_time diff --git a/mission-control/docs/guide/audit/access-logs.mdx b/mission-control/docs/guide/audit/access-logs.mdx index 0d825b33..464d9201 100644 --- a/mission-control/docs/guide/audit/access-logs.mdx +++ b/mission-control/docs/guide/audit/access-logs.mdx @@ -7,25 +7,61 @@ sidebar_custom_props: import {Card, Cards} from '@site/../common/src/components/Card' -Access logs track who accessed configuration items and when. This enables compliance auditing, security monitoring, and access reviews for your infrastructure. +Config access tracking has two parts: + +- **Config Access** — records who _has_ access to a config item (permissions, role assignments, group memberships) +- **Access Logs** — records who _actually accessed_ a config item and when + +Both are scraped alongside configuration data and feed into [Application](/docs/guide/audit/applications) access control views and [Identity & Access](/docs/guide/audit/identity-access) reports. ## Overview -When scraping configurations from external systems, you can also capture access logs that record: +When scraping configurations from external systems, you can capture: -- **Who** accessed a resource (external user) +- **Who** has access or accessed a resource (users, roles, groups) - **What** was accessed (config item) - **When** the access occurred - **How** they authenticated (MFA status, properties) -Access logs are stored separately from configuration data and can be queried independently for audit purposes. They feed into [Application](/docs/guide/audit/applications) access control views and [Identity & Access](/docs/guide/audit/identity-access) reports. +## Input Fields + +When providing `config_access` or `access_logs` in scraper output, the following shorthand fields are available alongside the standard schema fields. These are resolved during scraping and do not appear in the stored data. + +### Shorthand Fields + +| Field | Applies To | Resolves To | Description | +|-------|-----------|-------------|-------------| +| `user` | `config_access`, `access_logs` | `external_user_aliases` | Matches against user aliases to resolve the user | +| `role` | `config_access` | `external_role_aliases` | Matches against role aliases to resolve the role | +| `group` | `config_access` | `external_group_aliases` | Matches against group aliases to resolve the group | +| `config_id` | both | `config_id` (UUID) or `external_config_id.external_id` (non-UUID) | If a valid UUID, used directly. If a string, treated as an external ID for lookup | +| `config_type` / `type` | both | `external_config_id.config_type` | Config type for external ID lookup | + +### Alternative JSON Keys + +The scraper accepts these alternative key names in the JSON output: + +| Standard Key | Alternative | +|-------------|-------------| +| `config_access` | `access` | +| `access_logs` | `logs` | + +### Config Reference Defaulting + +When entries in `config_access` or `access_logs` don't specify a config reference, they inherit from the parent config object's top-level fields: + +- `uuid` or `config_id` → used as the config UUID +- `external_id` or `id` → used as the external ID +- `type` or `config_type` → used as the config type + +This means if the parent object has `"id": "db-prod-001"` and `"type": "Database"`, all access entries without their own config reference automatically target that config item. ## Access Log Schema + +:::info Deduplication +Access logs are deduplicated by the composite key `(config_id, external_user_id, scraper_id)`. When a duplicate is found, the `count` is incremented and `created_at`, `mfa`, and `properties` are updated only if the new record has a more recent `created_at`. +::: + +## Config Access Schema + +Records which users, roles, and groups have access to specific configuration items. At least one of `user`, `role`, or `group` (or their resolved UUID equivalents) must be set. + + @@ -111,15 +199,18 @@ All approaches require an **Entra ID P1 or P2 license**. Free-tier tenants do no For longer retention (beyond 30 days), export logs via Azure Monitor Diagnostic Settings to a log backend and use the [Logs scraper](/docs/integrations/azure-ad/audit-logs/logs-scraper) approach. -## Example: Custom Scraper with Access Logs +## Examples :::note -The `config_id` and `user_id` fields reference UUIDs in the database, but scrapers resolve them by matching on external identifiers (names, aliases, or scraper-assigned IDs). The example below uses human-readable strings to show the scraper input format — Mission Control resolves these to UUIDs internally. +The `config_id` and `user` fields reference UUIDs in the database, but scrapers resolve them by matching on external identifiers (names, aliases, or scraper-assigned IDs). The examples below use human-readable strings to show the scraper input format — Mission Control resolves these to UUIDs internally. ::: +### Access Logs + ```json title="config-with-access-logs.json" { "id": "db-prod-001", + "type": "Database", "config": { "name": "production-database", "engine": "postgres", @@ -127,7 +218,7 @@ The `config_id` and `user_id` fields reference UUIDs in the database, but scrape }, "access_logs": [ { - "config_id": "db-prod-001", + // highlight-next-line "user": "user-123", "created_at": "2025-01-08T10:30:00Z", "mfa": true, @@ -137,7 +228,6 @@ The `config_id` and `user_id` fields reference UUIDs in the database, but scrape } }, { - "config_id": "db-prod-001", "user": "user-456", "created_at": "2025-01-08T11:45:00Z", "mfa": false @@ -146,6 +236,34 @@ The `config_id` and `user_id` fields reference UUIDs in the database, but scrape } ``` +Both entries inherit `config_id` from the parent object's `id: "db-prod-001"` and `type: "Database"` — no need to repeat them. The `user` shorthand resolves against user aliases. + +### Config Access + +```json title="config-with-access.json" +{ + "id": "db-prod-001", + "type": "Database", + "config": { + "name": "production-database", + "engine": "postgres" + }, + "config_access": [ + { + // highlight-start + "user": "alice@example.com", + "role": "db-admin" + // highlight-end + }, + { + "group": "dba-team", + "config_id": "db-prod-001", + "config_type": "Database" + } + ] +} +``` + ## Compliance Use Cases Access logs provide the audit trail required for compliance frameworks: diff --git a/mission-control/docs/guide/config-db/concepts/changes.md b/mission-control/docs/guide/config-db/concepts/changes.md index ed0e4c65..66f97c43 100644 --- a/mission-control/docs/guide/config-db/concepts/changes.md +++ b/mission-control/docs/guide/config-db/concepts/changes.md @@ -33,12 +33,18 @@ When `full: true` is set, the scraper expects each config item to have these top - `config` - The actual configuration data to store - `changes` - An array of change events -- `access_logs` - Access log entries (see [Access Logs](/docs/guide/audit/access-logs)) +- `config_access` (or `access`) - Config access records (see [Access Logs](/docs/guide/audit/access-logs)) +- `access_logs` (or `logs`) - Access log entries (see [Access Logs](/docs/guide/audit/access-logs)) +- `analysis` - Analysis results :::note Fields other than these are ignored. Missing fields are treated as empty. ::: +:::tip Config Reference Defaulting +Top-level fields like `id`/`external_id`, `uuid`/`config_id`, and `type`/`config_type` are used as defaults for `changes`, `access_logs`, `config_access`, and `analysis` entries that don't specify their own config reference. +::: + ### Example Consider a file containing: diff --git a/mission-control/docs/guide/config-db/scrapers/custom.mdx b/mission-control/docs/guide/config-db/scrapers/custom.mdx deleted file mode 100644 index 7cf53317..00000000 --- a/mission-control/docs/guide/config-db/scrapers/custom.mdx +++ /dev/null @@ -1,301 +0,0 @@ ---- -title: Custom Scraper -sidebar_custom_props: - icon: hugeicons:code -sidebar_position: 20 -description: Full scraper output schema for custom scrapers with users, groups, roles, access logs, and alias resolution ---- - -import Fields from '@site/src/components/Fields' - -When you enable `full: true`, custom scrapers can return complex objects containing config data, changes, access logs, users, groups, and roles. This enables IAM scraping and compliance auditing. - -For transforming scraped configs before they are saved (field exclusions, masking, relationships, etc.), see the [Transformation Reference](/docs/reference/config-db/transformation). - -## Top-Level Fields - -With `full: true`, the scraper expects each item to have these top-level fields. - -| Field | Schema | Description | -|-------|--------|-------------| -| `config` | object | The actual configuration data to store | -| `changes` | [][Change](/docs/guide/config-db/concepts/changes#extracting-changes) | Change events | -| `logs` | [][AccessLog](#access-log) | Access log entries | -| `access` | [][ConfigAccess](#config-access) | Access permissions linking users/groups/roles to configs | -| `users` | [][User](#user) | User definitions from identity systems | -| `groups` | [][Group](#group) | Group definitions from identity systems | -| `roles` | [][Role](#role) | Role definitions from identity systems | -| `user_groups` | [][UserGroup](#user-group) | User-to-group membership mappings | - -Additional top-level fields for config identification: - -| Field | Description | -|-------|-------------| -| `id` | Unique identifier for the config item | -| `external_id` | External identifier for the config (used by `access_logs` and `config_access` to resolve config references) | -| `config_type` or `type` | Config type (used alongside `external_id` for config reference resolution) | -| `uuid` or `config_id` | UUID of the config item (used for direct config reference) | - -## User - - - -## Group - - - -## Role - - - -## User Group - - - -## Config Access - -The `access` array links users, groups, and roles to specific config items. - - - -At least one of `user`, `group`, or `role` must be set. - -## Access Log - -The `logs` array records individual access events. - - - -`user` must be set. - -## Alias Resolution - -When processing `access` and `logs` entries, the scraper resolves aliases to actual entity IDs: - -1. `user` is matched against the `aliases` field of each `users` entry -2. `group` is matched against the `aliases` field of each `groups` entry -3. `role` is matched against the `aliases` field of each `roles` entry - -If no matching entity is found, one is auto-created with the alias as its name. - -### Config Reference Resolution - -Config items in `access` and `logs` can be referenced by `config_id` — either a UUID pointing directly to a config item, or a non-UUID string treated as an external ID. - -If not provided, the top-level `id`/`external_id`/`config_type` fields are used as defaults. - -## Example - -```json title="config-with-access.json" -{ - "id": "test-org-role-access", - "config": { - "name": "Test Organization", - "type": "organization" - }, - "users": [ - { - "name": "Charlie Brown", - "account_id": "org-789", - "user_type": "human", - "email": "charlie@example.com", - "aliases": ["charlie-brown", "charlie@example.com"] - } - ], - "roles": [ - { - "name": "Editor", - "account_id": "org-789", - "role_type": "custom", - "description": "Edit access", - "aliases": ["editor-role", "edit-access"] - } - ], - "groups": [ - { - "name": "Editors Group", - "account_id": "org-789", - "group_type": "team", - "aliases": ["editors-group", "edit-team"] - } - ], - "access": [ - { - "id": "role-access-001", - "external_config_id": { - "config_type": "Organization", - "external_id": "test-org-role-access" - }, - "user": "charlie-brown", - "role": "editor-role", - "group": "editors-group" - } - ], - "changes": [ - { - "change_type": "permission_grant", - "summary": "Charlie granted editor access", - "created_at": "2025-01-08T10:00:00Z" - } - ] -} -``` - -## Extracting Changes & Access Logs - -When you enable `full: true`, custom scrapers can ingest [changes](/docs/guide/config-db/concepts/changes#extracting-changes) and [access logs](/docs/guide/audit/access-logs) from external systems by separating the config data from change events in your source. diff --git a/mission-control/docs/guide/config-db/scrapers/custom/access-mapping.mdx b/mission-control/docs/guide/config-db/scrapers/custom/access-mapping.mdx new file mode 100644 index 00000000..0af59534 --- /dev/null +++ b/mission-control/docs/guide/config-db/scrapers/custom/access-mapping.mdx @@ -0,0 +1,279 @@ +--- +title: Access Mapping +sidebar_position: 2 +sidebar_custom_props: + icon: mdi:account-lock +description: Scrape users, groups, roles, access permissions, and access logs from custom sources +--- + +import Fields from '@site/src/components/Fields' + +When `full: true` is set, custom scrapers can import identity and access data alongside config items. This includes: + +- **Users, Groups, Roles** — identity entities from external systems +- **Config Access** (`access`) — who _has_ access to a config item +- **Access Logs** (`logs`) — who _actually accessed_ a config item + +This data feeds into [Identity & Access](/docs/guide/audit/identity-access) reports and [Application](/docs/guide/audit/applications) views. + +## Example + +```json title="config-with-access.json" +{ + "id": "db-prod-001", + "type": "Database", + "config": { + "engine": "postgres", + "version": "15.2" + }, + // highlight-start + "users": [ + { + "name": "Alice Smith", + "account_id": "org-001", + "user_type": "human", + "email": "alice@example.com", + "aliases": ["alice@example.com", "asmith"] + } + ], + "roles": [ + { + "name": "Database Admin", + "account_id": "org-001", + "role_type": "custom", + "aliases": ["db-admin"] + } + ], + "access": [ + { + "user": "alice@example.com", + "role": "db-admin" + } + ], + "logs": [ + { + "user": "alice@example.com", + "created_at": "2025-01-08T10:30:00Z", + "mfa": true, + "properties": { + "ip_address": "192.168.1.100", + "client": "psql" + } + } + ] + // highlight-end +} +``` + +The `user`, `role`, and `group` fields in `access` and `logs` entries are **shorthands** — they're matched against the `aliases` of entities defined in the `users`, `roles`, and `groups` arrays. If no match is found, a new entity is auto-created. + +No `config_id` is needed in the `access` and `logs` entries — they inherit from the parent object's `id` and `type` fields. See [Config Reference Defaulting](../custom#config-reference-defaulting). + +## Identity Entities + +### User + + + +### Group + + + +### Role + + + +### User Group + +Links users to group memberships. + + + +## Config Access + +The `access` (or `config_access`) array records who has access to a config item. + + + +At least one of `user`, `role`, or `group` must be set. At least one of `scraper_id` (auto-populated), `application_id`, or `source` must be set. + +## Access Logs + +The `logs` (or `access_logs`) array records individual access events. + + + +:::info Deduplication +Access logs are deduplicated by `(config_id, user, scraper_id)`. On conflict, `count` is incremented and `created_at`, `mfa`, `properties` are updated only if the new record is more recent. +::: + +## Related + +- [Access Logs Guide](/docs/guide/audit/access-logs) — compliance use cases, Entra ID integration +- [Config Access Reference](/docs/reference/config-db/config_access) — full entity model and DB schema +- [Identity & Access](/docs/guide/audit/identity-access) — access review dashboards diff --git a/mission-control/docs/guide/config-db/scrapers/custom/changes.mdx b/mission-control/docs/guide/config-db/scrapers/custom/changes.mdx new file mode 100644 index 00000000..87fbfdf4 --- /dev/null +++ b/mission-control/docs/guide/config-db/scrapers/custom/changes.mdx @@ -0,0 +1,138 @@ +--- +title: Changes +sidebar_position: 1 +sidebar_custom_props: + icon: octicon:diff-16 +description: Ingest change events from custom scrapers using the full extraction mode +--- + +import Fields from '@site/src/components/Fields' + +When `full: true` is set, each scraped item can include a `changes` array of change events. These are recorded separately from the config data and appear in the config item's change history. + +See [Changes](/docs/guide/config-db/concepts/changes) for background on how Mission Control tracks changes. + +## Example + +```json title="config-with-changes.json" +{ + "id": "db-prod-001", + "type": "Database", + "config": { + "engine": "postgres", + "version": "15.2" + }, + // highlight-start + "changes": [ + { + "change_type": "upgrade", + "summary": "Upgraded PostgreSQL from 15.1 to 15.2", + "severity": "low", + "created_at": "2025-01-08T10:00:00Z", + "created_by": "alice@example.com", + "external_change_id": "CR-2025-0108" + }, + { + "change_type": "config_change", + "summary": "Increased max_connections from 100 to 200", + "created_at": "2025-01-07T14:30:00Z", + "details": { + "parameter": "max_connections", + "old_value": "100", + "new_value": "200" + } + } + ] + // highlight-end +} +``` + +With a scraper: + +```yaml +apiVersion: configs.flanksource.com/v1 +kind: ScrapeConfig +metadata: + name: database-changes +spec: + # highlight-next-line + full: true + file: + - type: Database + id: $.id + paths: + - databases/*.json +``` + +The scraper stores only the `config` object and records each change event on the config item. + +## Change Schema + + + +## Related + +- [Changes Concepts](/docs/guide/config-db/concepts/changes) — diff vs event-based changes, change traversal +- [Transformation Changes](/docs/guide/config-db/concepts/transform#changes) — change exclusions, mapping, and severity overrides diff --git a/mission-control/docs/guide/config-db/scrapers/custom/index.mdx b/mission-control/docs/guide/config-db/scrapers/custom/index.mdx new file mode 100644 index 00000000..541d6532 --- /dev/null +++ b/mission-control/docs/guide/config-db/scrapers/custom/index.mdx @@ -0,0 +1,112 @@ +--- +title: Custom Scraper +sidebar_custom_props: + icon: hugeicons:code +sidebar_position: 20 +description: Scrape config items from any source using file, exec, HTTP, or SQL scrapers with JSONPath mapping +--- + +import Custom from '../_custom.mdx' +import {Card, Cards} from '@site/../common/src/components/Card' +import CustomScraperDiagram from '@site/src/components/CustomScraperDiagram' + +Custom scrapers let you import config items from any source — files, scripts, HTTP APIs, or SQL queries. You define how to extract each item's identity (`id`, `type`, `name`) from the raw data using JSONPath or static values. + + + +## Mapping + + + +## Example: Backstage Catalog + +This scraper checks out the [Backstage](https://backstage.io) repository and imports its catalog entities (Components, APIs, Systems, Domains) as config items with relationships. + +```yaml title='exec-backstage-catalog.yaml' file=/modules/config-db/fixtures/exec-backstage-catalog.yaml + +``` + +The Python script reads Backstage YAML catalog files and outputs a JSON array. Each item maps to a config item via the `id`, `type`, and `name` fields in the scraper spec. + +### Relationships + +The `transform.relationship` expressions create links between catalog items using [CEL](/docs/reference/scripting/cel): + +- **Component → System** via `spec.system` +- **Component → API** via `spec.consumesApis` and `spec.dependsOn` +- **Component/System/Domain → Owner** via `spec.owner` +- **System → Domain** via `spec.domain` + +See [Relationships](/docs/guide/config-db/concepts/relationships) and [Transformation](/docs/guide/config-db/concepts/transform) for more details. + +## Full Mode + +When you set `full: true`, each scraped item can include additional data beyond just the config: + +```yaml +spec: + # highlight-next-line + full: true + file: + - type: $.type + id: $.id + paths: + - configs/*.json +``` + +With `full: true`, the scraper expects each item to be a JSON object with these top-level fields: + +| Field | Alternative Key | Description | +|-------|----------------|-------------| +| `config` | | The actual configuration data to store | +| `changes` | | Change events to record on this config item | +| `config_access` | `access` | Access permissions (who _has_ access) | +| `access_logs` | `logs` | Access log events (who _actually accessed_) | +| `analysis` | | Insights (security, cost, performance findings) | +| `users` | `external_users` | User definitions from identity systems | +| `groups` | `external_groups` | Group definitions | +| `roles` | `external_roles` | Role definitions | +| `user_groups` | `external_user_groups` | User-to-group memberships | + +Fields other than these are ignored. Missing fields are treated as empty. + +### Config Reference Defaulting + +Top-level identification fields are used as defaults for child entries (`changes`, `access_logs`, `config_access`, `analysis`) that don't specify their own config reference: + +| Field | Alternative | +|-------|------------| +| `uuid` | `config_id` | +| `external_id` | `id` | +| `type` | `config_type` | + +```json title="example.json" +{ + // highlight-start + "id": "db-prod-001", + "type": "Database", + // highlight-end + "config": { "engine": "postgres", "version": "15.2" }, + "changes": [ + { + // No config reference needed — inherits id + type from above + "change_type": "upgrade", + "summary": "Upgraded to 15.2" + } + ], + "access": [ + { "user": "alice@example.com", "role": "db-admin" } + ], + "logs": [ + { "user": "alice@example.com", "created_at": "2025-01-08T10:30:00Z" } + ] +} +``` + +## Sub-Pages + + + + + + diff --git a/mission-control/docs/guide/config-db/scrapers/custom/insights.mdx b/mission-control/docs/guide/config-db/scrapers/custom/insights.mdx new file mode 100644 index 00000000..c55ae089 --- /dev/null +++ b/mission-control/docs/guide/config-db/scrapers/custom/insights.mdx @@ -0,0 +1,116 @@ +--- +title: Insights +sidebar_position: 3 +sidebar_custom_props: + icon: fluent-mdl2:insights +description: Attach security, cost, and performance insights to config items from custom scrapers +--- + +import Fields from '@site/src/components/Fields' + +When `full: true` is set, each scraped item can include an `analysis` array of insight findings. These appear as insights on the config item and can be filtered by type, severity, and analyzer. + +See [Insights](/docs/guide/config-db/concepts/insights) for the built-in insight providers (Trivy, AWS Trusted Advisor, Azure Advisor). + +## Example + +```json title="config-with-insights.json" +{ + "id": "i-0abc123def456", + "type": "EC2Instance", + "config": { + "instance_type": "m5.4xlarge", + "state": "running" + }, + // highlight-start + "analysis": [ + { + "analyzer": "cost-optimization", + "analysis_type": "cost", + "severity": "medium", + "summary": "Instance is underutilized — average CPU usage is 5% over 14 days", + "source": "AWS Trusted Advisor", + "first_observed": "2025-01-01T00:00:00Z", + "last_observed": "2025-01-08T00:00:00Z", + "status": "open", + "messages": [ + "Consider downsizing to m5.xlarge", + "Estimated monthly savings: $200" + ] + }, + { + "analyzer": "security", + "analysis_type": "security", + "severity": "high", + "summary": "Instance has a public IP with unrestricted SSH access", + "source": "Security Scanner" + } + ] + // highlight-end +} +``` + +## Analysis Schema + + + +## Related + +- [Insights Concepts](/docs/guide/config-db/concepts/insights) — built-in insight providers diff --git a/mission-control/docs/guide/mcp/client-setup.mdx b/mission-control/docs/guide/mcp/client-setup.mdx index 80af540e..fbd43425 100644 --- a/mission-control/docs/guide/mcp/client-setup.mdx +++ b/mission-control/docs/guide/mcp/client-setup.mdx @@ -137,9 +137,7 @@ Use the Setup MCP flow to create an access token with MCP permissions. :::info -Some MCP clients use the access token as a bearer token. Other MCP clients use Basic authentication with username `token` and password ``. - -When you use the `Authorization: Basic ` header, encode `token:` as the `base64_token` value. +Use the access token as a bearer token: `Authorization: Bearer `. ::: ### Configure a client @@ -235,7 +233,7 @@ GitHub Copilot Chat in VS Code supports MCP through the Copilot extensibility AP "https://mc..workload-prod-eu-02.flanksource.com/mcp" ], "env": { - "AUTHORIZATION": "Basic YOUR_TOKEN_HERE" + "AUTHORIZATION": "Bearer YOUR_TOKEN_HERE" } } } @@ -274,7 +272,7 @@ Cline supports MCP through its settings configuration. "https://mc..workload-prod-eu-02.flanksource.com/mcp" ], "env": { - "AUTHORIZATION": "Basic YOUR_TOKEN_HERE" + "AUTHORIZATION": "Bearer YOUR_TOKEN_HERE" } } } @@ -309,7 +307,7 @@ Continue.dev supports MCP servers through its configuration file. "https://mc..workload-prod-eu-02.flanksource.com/mcp" ], "env": { - "AUTHORIZATION": "Basic YOUR_TOKEN_HERE" + "AUTHORIZATION": "Bearer YOUR_TOKEN_HERE" } } ] @@ -343,7 +341,7 @@ Zed supports MCP through its assistant configuration. "https://mc..workload-prod-eu-02.flanksource.com/mcp" ], "env": { - "AUTHORIZATION": "Basic YOUR_TOKEN_HERE" + "AUTHORIZATION": "Bearer YOUR_TOKEN_HERE" } } } @@ -364,13 +362,13 @@ For custom integrations or clients that support direct HTTP connections: - **Base URL**: `https://mc..workload-prod-eu-02.flanksource.com/mcp` - **Protocol**: HTTP with SSE (Server-Sent Events) -- **Authentication**: Basic token in Authorization header +- **Authentication**: Bearer token in the Authorization header ### Example Request ```bash curl -X POST https://mc..workload-prod-eu-02.flanksource.com/mcp \ - -H "Authorization: Basic YOUR_TOKEN_HERE" \ + -H "Authorization: Bearer YOUR_TOKEN_HERE" \ -H "Content-Type: application/json" \ -d '{ "jsonrpc": "2.0", @@ -392,7 +390,7 @@ const ws = new WebSocket( 'wss://mc..workload-prod-eu-02.flanksource.com/mcp/ws', { headers: { - Authorization: 'Basic YOUR_TOKEN_HERE' + Authorization: 'Bearer YOUR_TOKEN_HERE' } } ) @@ -427,7 +425,7 @@ ws.on('open', () => { 2. **Authentication Failed** - Verify token is valid and not expired - Ensure token has `mcp.*` permissions - - Check if Basic prefix is included + - Check if the Bearer prefix is included 3. **Tools Not Appearing** - Restart the client application @@ -442,7 +440,7 @@ You can test your MCP connection using the MCP CLI: npx @modelcontextprotocol/cli connect \ --transport http \ --url https://mc..workload-prod-eu-02.flanksource.com/mcp \ - --header "Authorization: Basic YOUR_TOKEN_HERE" + --header "Authorization: Bearer YOUR_TOKEN_HERE" ``` diff --git a/mission-control/docs/guide/mcp/index.mdx b/mission-control/docs/guide/mcp/index.mdx index ad51e9bf..af251452 100644 --- a/mission-control/docs/guide/mcp/index.mdx +++ b/mission-control/docs/guide/mcp/index.mdx @@ -10,7 +10,7 @@ import DocCardList from '@theme/DocCardList'; # MCP -Mission Control supports MCP for those who want to link it with their LLM Clients. +Mission Control supports [MCP (Model Context Protocol)](https://modelcontextprotocol.io/) for connecting AI assistants to your infrastructure. See the [MCP integration overview](/docs/integrations/mcp) for a high-level introduction. ## Setup @@ -24,6 +24,51 @@ For detailed setup instructions for popular MCP clients, see [Client Setup](./cl ## Permissions -To access the mcp server, `Authorization` header can be used with a token. The token must have `mcp.*` permission. +Access to the MCP server requires a token with the `mcp:run` permission, passed via the `Authorization` header. + +- **`mcp:run`** is the single permission that controls access to all MCP tools — there is no per-tool granularity +- **Admin users** bypass the permission check entirely +- **Deny support**: A permission with `Deny: true` blocks MCP access even if another grant exists +- **Row-Level Security (RLS)**: All tool results are filtered based on the user's existing RBAC scopes (namespace, team, tags). Users only see catalog items, playbooks, views, and health checks they already have access to + +### Creating a Token + +The easiest way to create a token is via the setup wizard: click your user avatar and choose **Setup MCP**. This generates a token with the correct permissions and provides ready-to-use client configuration snippets. + +You can also create tokens via Settings → API Tokens, ensuring the token has `mcp:run` permission. + +:::info +Use the token as a Bearer token: `Authorization: Bearer ` +::: + +## What's Available + +### Tools + +Mission Control exposes 20+ static tools plus dynamic tools generated from your playbooks and views: + +| Category | Examples | +|----------|---------| +| [Catalog](/docs/guide/mcp/tools/catalog_tools) | `search_catalog`, `describe_catalog`, `search_catalog_changes`, `get_related_configs`, `list_catalog_types` | +| [Health Checks](/docs/guide/mcp/tools) | `search_health_checks`, `get_check_status`, `run_health_check`, `list_all_checks` | +| [Playbooks](/docs/guide/mcp/tools/playbook_tools) | `get_all_playbooks`, `get_playbook_recent_runs`, `get_playbook_failed_runs`, `get_playbook_run_steps` | +| Views | `list_all_views` + per-view query tools (synced hourly) | +| [Connections](/docs/guide/mcp/tools/connection_tools) | `list_connections` | +| Artifacts | `read_artifact_metadata`, `read_artifact_content` | +| Notifications | `get_notification_detail`, `get_notifications_for_resource` | +| Templates | `run_template` | + +**Dynamic tools**: Each playbook generates an execution tool (e.g. `deploy-app_prod_incident-response`). Each view generates a query tool (e.g. `view_incidents_prod`). These are registered per-session for playbooks and synced hourly for views. + +### Resources + +MCP resources provide direct access to specific items via URI templates: + +| Resource | URI Template | +|----------|-------------| +| [Config Item](/docs/guide/mcp/resources/config_item) | `config_item://{id}` | +| [Connection](/docs/guide/mcp/resources/connection) | `connection://{namespace}/{name}` | +| [Playbook](/docs/guide/mcp/resources/playbooks) | `playbook://{idOrName}` | +| View | `view://{namespace}/{name}` | diff --git a/mission-control/docs/guide/mcp/resources/index.mdx b/mission-control/docs/guide/mcp/resources/index.mdx index 35ec3e2f..a7a68cb5 100644 --- a/mission-control/docs/guide/mcp/resources/index.mdx +++ b/mission-control/docs/guide/mcp/resources/index.mdx @@ -9,6 +9,15 @@ import DocCardList from '@theme/DocCardList'; # MCP Resources -Mission Control provides various resources through the MCP interface that can be accessed using URI templates. +Mission Control exposes the following resources through MCP. Clients can access them using URI templates. - \ No newline at end of file +| Resource | URI Template | Description | +|----------|-------------|-------------| +| [Config Item](./config_item.mdx) | `config_item://{id}` | Retrieve detailed config item data by UUID | +| [Connection](./connection.mdx) | `connection://{namespace}/{name}` | Retrieve connection configuration details | +| [Playbook](./playbooks.mdx) | `playbook://{idOrName}` | Get playbook parameters and metadata | +| View | `view://{namespace}/{name}` | Get view definition and metadata | + +All resources return `application/json`. + + diff --git a/mission-control/docs/guide/mcp/resources/playbooks.mdx b/mission-control/docs/guide/mcp/resources/playbooks.mdx index 913c733e..29be5cb8 100644 --- a/mission-control/docs/guide/mcp/resources/playbooks.mdx +++ b/mission-control/docs/guide/mcp/resources/playbooks.mdx @@ -10,7 +10,7 @@ Get playbook with their parameters and metadata. ## URI Template -`playbooks://{id}` +`playbook://{idOrName}` **MIME Type:** `application/json` @@ -22,7 +22,7 @@ This resource allows you to retrieve detailed information about a specific playb ### Sample Request -**URI:** `playbooks://restart-pod` +**URI:** `playbook://restart-pod` ### Sample Response @@ -63,7 +63,7 @@ This resource allows you to retrieve detailed information about a specific playb ### Sample Request (Complex Playbook) -**URI:** `playbooks://database-backup` +**URI:** `playbook://database-backup` ### Sample Response diff --git a/mission-control/docs/guide/mcp/tools/catalog_tools.mdx b/mission-control/docs/guide/mcp/tools/catalog_tools.mdx index b6332cb2..18205baa 100644 --- a/mission-control/docs/guide/mcp/tools/catalog_tools.mdx +++ b/mission-control/docs/guide/mcp/tools/catalog_tools.mdx @@ -20,7 +20,7 @@ List all available configuration types in the system **Result:** Returns a list of all configuration types like `AWS::EC2::Instance`, `Kubernetes::Pod`, `Azure::VM::Instance`, etc. -## catalog_search +## search_catalog Search across the entire configuration catalog @@ -53,7 +53,7 @@ Search across the entire configuration catalog **Result:** Returns recent pods with full resource data -## catalog_changes_search +## search_catalog_changes Search for configuration changes across the catalog @@ -64,7 +64,7 @@ Search for configuration changes across the catalog **Features:** - Search by change-specific fields like severity, change type, and summary -- Same query syntax as catalog_search with additional change-related fields +- Same query syntax as search_catalog with additional change-related fields ### Example Usage @@ -86,7 +86,7 @@ Search for configuration changes across the catalog **Result:** Returns the 10 most recent changes to any AWS resources including the change type, summary, and affected resources. -## related_configs +## get_related_configs Find configurations related to a specific config item diff --git a/mission-control/docs/guide/mcp/tools/index.mdx b/mission-control/docs/guide/mcp/tools/index.mdx index 51605782..aef78a7e 100644 --- a/mission-control/docs/guide/mcp/tools/index.mdx +++ b/mission-control/docs/guide/mcp/tools/index.mdx @@ -9,6 +9,72 @@ import DocCardList from '@theme/DocCardList'; # MCP Tools -Mission Control provides various tools through the MCP interface to interact with different aspects of the system. +Mission Control provides the following tools through MCP. Tool names and descriptions are discoverable by any MCP client via the protocol's `tools/list` method. - \ No newline at end of file +## Catalog + +| Tool | Description | Read-Only | +|------|-------------|-----------| +| `search_catalog` | Search and find configuration items in the catalog | Yes | +| `describe_catalog` | Get all data for specific config items | Yes | +| `search_catalog_changes` | Search for configuration change events | Yes | +| `get_related_configs` | Find configs related to a specific config item | Yes | +| `list_catalog_types` | List all available configuration types | Yes | + +## Health Checks + +| Tool | Description | Read-Only | +|------|-------------|-----------| +| `search_health_checks` | Search and find health checks | Yes | +| `get_check_status` | Get health check execution history | Yes | +| `list_all_checks` | List all health checks with complete metadata | Yes | +| `run_health_check` | Execute a health check immediately | No | + +## Playbooks + +| Tool | Description | Read-Only | +|------|-------------|-----------| +| `get_all_playbooks` | List all available playbooks | Yes | +| `get_playbook_recent_runs` | Get recent playbook execution history | Yes | +| `get_playbook_failed_runs` | Get recent failed playbook runs | Yes | +| `get_playbook_run_steps` | Get detailed information about a specific playbook run | Yes | +| _Per-playbook tools_ | Each playbook generates an execution tool (e.g. `deploy-app_prod_ops`) | No | + +Per-playbook tools are registered dynamically at session start. Their names follow the pattern `{name}_{namespace}_{category}` and their input schemas are derived from the playbook's parameter spec. + +## Views + +| Tool | Description | Read-Only | +|------|-------------|-----------| +| `list_all_views` | List all available view tools | Yes | +| _Per-view tools_ | Each view generates a query tool (e.g. `view_incidents_prod`) | Yes | + +Per-view tools are synced every hour. Their names follow the pattern `view_{name}_{namespace}`. + +## Connections + +| Tool | Description | Read-Only | +|------|-------------|-----------| +| `list_connections` | List all configured connections | Yes | + +## Artifacts + +| Tool | Description | Read-Only | +|------|-------------|-----------| +| `read_artifact_metadata` | Get artifact metadata by ID | Yes | +| `read_artifact_content` | Read the content of an artifact file | Yes | + +## Notifications + +| Tool | Description | Read-Only | +|------|-------------|-----------| +| `get_notification_detail` | Get detailed information about a specific notification | Yes | +| `get_notifications_for_resource` | Get notification history for a specific resource | Yes | + +## Templates + +| Tool | Description | Read-Only | +|------|-------------|-----------| +| `run_template` | Evaluate a CEL expression or Go template | No | + + diff --git a/mission-control/docs/guide/mcp/tools/playbook_tools.mdx b/mission-control/docs/guide/mcp/tools/playbook_tools.mdx index 39142a0d..b508f177 100644 --- a/mission-control/docs/guide/mcp/tools/playbook_tools.mdx +++ b/mission-control/docs/guide/mcp/tools/playbook_tools.mdx @@ -8,7 +8,7 @@ sidebar_custom_props: Tools for working with playbooks in Mission Control. -## playbook_list_all +## get_all_playbooks List all available playbooks @@ -60,7 +60,7 @@ List all available playbooks ] ``` -## playbook_recent_runs +## get_playbook_recent_runs Get recent playbook execution runs @@ -97,7 +97,7 @@ Get recent playbook execution runs ] ``` -## playbook_failed_runs +## get_playbook_failed_runs Get recent failed playbook execution runs @@ -131,16 +131,16 @@ Get recent failed playbook execution runs ] ``` -## playbook_exec_run +## Dynamic Playbook Tools -Execute a playbook with specified parameters +Each playbook generates a dedicated execution tool at session start. The tool name follows the pattern `{name}_{namespace}_{category}` (e.g. `deploy-app_prod_ops`). Parameters are derived from the playbook's spec. -**Note:** This is a destructive operation that will modify system state. Parameters are playbook-specific and can be found using the playbook_list_all tool. +**Note:** This is a destructive operation that will modify system state. Parameters are playbook-specific and can be found using the `get_all_playbooks` tool. ### Example Usage diff --git a/mission-control/docs/guide/playbooks/actions/index.md b/mission-control/docs/guide/playbooks/actions/index.md index 4cfdaa36..af230df2 100644 --- a/mission-control/docs/guide/playbooks/actions/index.md +++ b/mission-control/docs/guide/playbooks/actions/index.md @@ -22,6 +22,7 @@ Actions are the fundamental tasks executed by a playbook. A playbook can compris | `sql` | Specify sql of action. | [Sql`](./sql) | | | `pod` | Specify pod of action. | [Pod](./pod) | | | `notification` | Specify notification of action. | [Notification](./notification) | | +| `report` | Generate reports from Views or config catalogs as JSON, CSV, HTML, PDF, Markdown, or Slack output | [Report](./report) | | :::note Specify one or more actions; but at least one. diff --git a/mission-control/docs/guide/playbooks/actions/logs.mdx b/mission-control/docs/guide/playbooks/actions/logs.mdx index 8362049a..c5d8995e 100644 --- a/mission-control/docs/guide/playbooks/actions/logs.mdx +++ b/mission-control/docs/guide/playbooks/actions/logs.mdx @@ -51,6 +51,8 @@ spec: {field: "mapping.severity", description: "Source field names for log level/severity", scheme: "[]string"}, {field: "mapping.message", description: "Source field names for log message content", scheme: "[]string"}, {field: "mapping.id", description: "Source field names for unique log identifier", scheme: "[]string"}, + {field: "mapping.groupBy", description: "Fields to group logs by; each unique combination of values becomes a separate log group", scheme: "[]string"}, + {field: "mapping.dedupBy", description: "Fields used to identify duplicate log entries for deduplication within groups", scheme: "[]string"}, {field: "cloudwatch", priority: 1, description: "Query logs from AWS CloudWatch Logs", scheme: "[CloudWatch](#cloudwatch)"}, {field: "kubernetes", priority: 1, description: "Fetch logs directly from Kubernetes pods", scheme: "[Kubernetes](#kubernetes)"}, {field: "loki", priority: 1, description: "Query logs from Grafana Loki", scheme: "[Loki](#loki)"}, diff --git a/mission-control/docs/guide/playbooks/actions/report.mdx b/mission-control/docs/guide/playbooks/actions/report.mdx new file mode 100644 index 00000000..f0e52828 --- /dev/null +++ b/mission-control/docs/guide/playbooks/actions/report.mdx @@ -0,0 +1,181 @@ +--- +title: Report +sidebar_custom_props: + icon: file-chart + +--- +import Templating from "@site/docs/reference/playbooks/context.mdx" + + + +# Report Action + +The Report action generates formatted output from Views or inline config catalog queries. Use it to produce JSON, CSV, HTML, PDF, Markdown, or Slack-formatted data for downstream actions like notifications or artifact storage. + +```yaml title="weekly-report.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: weekly-compliance-report +spec: + on: + schedule: + - schedule: "0 9 * * MON" + actions: + - name: generate-report + report: + view: default/compliance-summary + format: facet-pdf + facet: + connection: connection://facet/renderer + pdfOptions: + pageSize: A4 + landscape: false + variables: + env: production + - name: send-report + notification: + connection: connection://smtp/company-email + title: "Weekly Compliance Report" + body: "Please find the weekly compliance report attached." + attachments: + - "$(actions.generate-report.artifacts[0])" +``` + + + +## Report + + + +## Facet + +Facet options configure remote HTML/PDF rendering via a [Facet connection](/docs/reference/connections/facet). + + + +## Output + +The report action produces a binary artifact accessible via `.actions..artifacts[0]` in subsequent actions. + +```yaml +actions: + - name: build-report + report: + view: default/infra-summary + format: pdf + - name: notify + notification: + connection: connection://slack/ops + message: "Report ready: $(actions.build-report.artifacts[0])" +``` + +
+ +```yaml title="configs-json-report.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: export-unhealthy-configs +spec: + actions: + - name: find-unhealthy + report: + configs: + health: unhealthy + types: + - Kubernetes::Deployment + format: json + variables: + cluster: production +``` + +
+ +
+ +```yaml title="html-email-report.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: daily-html-report +spec: + on: + schedule: + - schedule: "@every 24h" + actions: + - name: generate + report: + view: default/daily-summary + format: html + variables: + date: "$(now | date \"2006-01-02\")" + - name: email + notification: + connection: connection://smtp/alerts + title: "Daily Summary Report" + body: "$(actions.generate.artifacts[0])" +``` + +
+ +
+ +```yaml title="scheduled-pdf-report.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: monthly-pdf-report +spec: + on: + schedule: + - schedule: "0 8 1 * *" + actions: + - name: render-pdf + report: + view: default/monthly-metrics + format: facet-pdf + facet: + connection: connection://facet/renderer + pdfOptions: + pageSize: A4 + landscape: true + margins: + top: 20 + bottom: 20 + left: 15 + right: 15 + header: | +
Monthly Report — $(now | date "January 2006")
+ footer: | +
Page of
+ timestampUrl: https://freetsa.org/tsr + variables: + month: "$(now | date \"2006-01\")" +``` + +
+ +## Templating + + diff --git a/mission-control/docs/guide/views/concepts/templating.md b/mission-control/docs/guide/views/concepts/templating.md index 8d2cbf74..458ca4c8 100644 --- a/mission-control/docs/guide/views/concepts/templating.md +++ b/mission-control/docs/guide/views/concepts/templating.md @@ -4,6 +4,8 @@ sidebar_custom_props: icon: mdi:variable --- +import ViewsVariablesDiagram from '@site/../common/src/components/ViewsVariablesDiagram' + Variables make views interactive by allowing users to dynamically control which data is fetched. Variables are defined in the `templating` section and their values are substituted into queries using the `$(var.)` syntax. ## Overview @@ -20,39 +22,7 @@ Variables in Mission Control allow you to: Mission Control has two ways to filter view data. Both are server-side, but they operate at different stages: -``` -┌─────────────────────────────────────┐ -│ Sources │ -│ (configs, prometheus, checks, etc) │ -└──────────────────┬──────────────────┘ - │ - ▼ - ┌──────────────────┐ - │ Execute Queries │ ◀── Variables filter here - └────────┬─────────┘ (only matching data is fetched) - │ - ▼ - ┌────────────────────┐ - │ In-memory SQLite │ - │ (joins/merge) │ - └──────────┬─────────┘ - │ - ▼ -┌─────────────────────────────────────┐ -│ PostgreSQL Table │ -│ (view__) │ -└──────────────────┬──────────────────┘ - │ - ▼ - ┌────────────────┐ - │ PostgREST API │ ◀── Column Filters add WHERE clauses here - └───────┬────────┘ - │ - ▼ - ┌──────────┐ - │ UI │ - └──────────┘ -``` + | | Variables | Column Filters | | -------------------------- | ---------------------------------------------------------- | -------------------------------------------------------- | diff --git a/mission-control/docs/guide/views/index.md b/mission-control/docs/guide/views/index.md index 7fcfc3e9..f81f5568 100644 --- a/mission-control/docs/guide/views/index.md +++ b/mission-control/docs/guide/views/index.md @@ -7,11 +7,12 @@ sidebar_custom_props: --- import View from "@site/docs/reference/views/\_view.mdx" +import ViewsDiagram from '@site/../common/src/components/ViewsDiagram' Views are dynamic, data-driven dashboards in Mission Control that aggregate and visualize data from multiple sources. They collect data from multiple sources into an in-memory SQLite database, enabling you to run any SQL query for filtering, joining, and aggregating your observability data. - + Views serve as a powerful data aggregation and visualization layer in Mission Control. They: diff --git a/mission-control/docs/installation/_properties_mission_control.mdx b/mission-control/docs/installation/_properties_mission_control.mdx index 361e0cfb..521aca69 100644 --- a/mission-control/docs/installation/_properties_mission_control.mdx +++ b/mission-control/docs/installation/_properties_mission_control.mdx @@ -9,8 +9,13 @@ import Container from './_properties_container.mdx' | image.pullPolicy | Defaults to `IfNotPresent` | | image.tag | | | otel.serviceName | Defaults to `mission-control` | +| `properties..disable` | Disable UI features when set to `"true"`. Supported features include `topology`, `health`, `incidents`, `config`, `logs`, `playbooks`, `applications`, `views`, `ai`, `agents`, and `settings.*`. | +| properties.dashboard.default.view | Backend dashboard view selector. Defaults to `mission-control-dashboard`. | +| properties.defaults.dashboard_view | UI sidebar dashboard navigation selector. | +| properties.flanksource.ui.snippets | Local JavaScript function expression executed once by the UI after user load. | | properties.incidents.disable | Defaults to `{}` | | properties.logs.disable | Defaults to `true` | +| properties.proxy.disable | In Clerk auth mode, force direct backend access instead of proxy access when set to `"true"`. | | kmsConnection | Provide the KMS connection string to use for secret parameters. See [KMS connection documentation](/docs/reference/connections/KMS/) for details. | | replicas | Defaults to `1` | | resources.limits.cpu | Defaults to `500m` | diff --git a/mission-control/docs/integrations/mcp.mdx b/mission-control/docs/integrations/mcp.mdx index 1ffcfdf3..bda494f1 100644 --- a/mission-control/docs/integrations/mcp.mdx +++ b/mission-control/docs/integrations/mcp.mdx @@ -6,91 +6,51 @@ sidebar_custom_props: import { Card, Cards } from '@site/../common/src/components/Card' import { CapabilityBadges, CapabilityBadge, CapabilityHeading } from '@site/../common/src/components/CapabilityBadges' +import MCPDiagram from '@site/../common/src/components/MCPDiagram' -Mission Control provides an MCP (Model Context Protocol) server that enables AI assistants like Claude to interact with your infrastructure. +Mission Control provides an [MCP](https://modelcontextprotocol.io/) server that enables AI assistants to interact with your infrastructure. The server is built into Mission Control and available at the `/mcp` endpoint — no separate service is required. -**Use cases:** -- Query the config catalog and explore infrastructure relationships -- List and inspect health checks across your environment -- Execute playbooks to remediate issues or perform operations -- Manage connections, artifacts, and notifications -- Enable AI-assisted troubleshooting and incident response + -## MCP Server +## Available Tools -Mission Control exposes an MCP server endpoint that AI assistants can connect to for infrastructure interactions. +Mission Control exposes **20+ tools** across these categories, plus **dynamic tools** generated from your playbooks and views: -### Available Tools +| Category | Tools | Description | +|----------|-------|-------------| +| **Catalog** | 5 | Search config items, describe configs, find related items, list types, search changes | +| **Health Checks** | 4 | Search checks, get status history, run checks, list all checks | +| **Playbooks** | 4 + dynamic | List playbooks, view runs, get run steps, plus a per-playbook execution tool for each playbook | +| **Views** | 1 + dynamic | List views, plus a per-view query tool synced hourly | +| **Connections** | 1 | List configured connections | +| **Artifacts** | 2 | Read artifact metadata and content | +| **Notifications** | 2 | Get notification details and history for a resource | +| **Templates** | 1 | Evaluate CEL expressions or Go templates | -| Tool | Description | -|------|-------------| -| **Catalog** | Query config items, search by type, and explore relationships | -| **Health Checks** | List canary health checks, view status and history | -| **Playbooks** | List available playbooks and trigger executions | -| **Connections** | Browse configured connections to external systems | -| **Artifacts** | List and retrieve artifacts from playbook runs | -| **Notifications** | View notification configurations | -| **Jobs** | Monitor scheduled jobs and their status | +See the [complete tools reference](/docs/guide/mcp/tools) for the full list. -### Connecting Claude Desktop +## Permissions -Add Mission Control to your Claude Desktop configuration: +Access requires the `mcp:run` permission. Admin users bypass this check. All data is filtered by Row-Level Security (RLS) — users only see resources matching their existing RBAC scopes. -```json title="claude_desktop_config.json" -{ - "mcpServers": { - "mission-control": { - "command": "npx", - "args": [ - "mcp-remote", - "https://your-mission-control.example.com/mcp", - "--header", - "Authorization: Bearer ${MISSION_CONTROL_TOKEN}" - ] - } - } -} -``` +See [MCP Permissions](/docs/guide/mcp#permissions) for details. -### Authentication +## Getting Started -The MCP server requires authentication with the `mcp:run` permission. Create an API token with appropriate RBAC permissions: - -```bash -kubectl create token mission-control-mcp --duration=8760h -``` - -### Example Interactions - -Once connected, you can ask Claude questions like: - -- "What health checks are failing right now?" -- "Show me all Kubernetes deployments in the production namespace" -- "Run the restart-pod playbook for the api-server deployment" -- "What config changes happened in the last hour?" - -## Architecture - -``` -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ Claude/AI │────▶│ MCP Server │────▶│ Mission Control│ -│ Assistant │◀────│ (SSE/HTTP) │◀────│ API │ -└─────────────────┘ └─────────────────┘ └─────────────────┘ -``` - -The MCP server translates natural language requests from AI assistants into Mission Control API calls, enabling conversational infrastructure management. - -## Next Steps +Create a token via the setup wizard (Settings → Setup MCP) and configure your AI client. - - Learn about automating operations with playbooks + + Step-by-step setup for Claude Desktop, Claude Code, VS Code, and more + + + Complete reference of available MCP tools - - Explore the configuration catalog + + MCP resource URI templates for direct data access diff --git a/mission-control/docs/reference/config-db/config_access.mdx b/mission-control/docs/reference/config-db/config_access.mdx index d5cfb856..2879ad57 100644 --- a/mission-control/docs/reference/config-db/config_access.mdx +++ b/mission-control/docs/reference/config-db/config_access.mdx @@ -31,9 +31,8 @@ Users represent individuals or service accounts from your identity providers or }, { field: "account_id", - description: "Account identifier from the source system", - scheme: "string", - required: true + description: "Account identifier from the source system. Auto-populated during scraping if not provided", + scheme: "string" }, { field: "user_type", @@ -52,9 +51,8 @@ Users represent individuals or service accounts from your identity providers or }, { field: "scraper_id", - description: "ID of the scraper that created this user", - scheme: "uuid", - required: true + description: "ID of the scraper that created this user. Auto-populated from scraper context", + scheme: "uuid" } ]}/> @@ -168,36 +166,57 @@ Links users to their group memberships. ## Config Access -Records which users, groups, and roles have access to specific configuration items. At least one of `external_user_id`, `external_group_id`, or `external_role_id` must be set. +Records which users, groups, and roles have access to specific configuration items. At least one of `external_user_id`, `external_group_id`, or `external_role_id` (or their shorthand equivalents `user`, `role`, `group`) must be set. + +At least one of `scraper_id`, `application_id`, or `source` must be set. When run via a scraper, `scraper_id` is auto-populated from context. +### Input Shorthands + +These fields are accepted in scraper input and resolved during processing: + +| Shorthand | Resolves To | Description | +|-----------|-------------|-------------| +| `user` | `external_user_aliases` | Matched against user aliases | +| `role` | `external_role_aliases` | Matched against role aliases | +| `group` | `external_group_aliases` | Matched against group aliases | +| `config_id` (non-UUID) | `external_config_id.external_id` | External ID for config lookup | +| `config_type` / `type` | `external_config_id.config_type` | Config type for external ID lookup | + +### ExternalID {#externalid} + + + ## Config Access Log -Records individual access events for audit purposes. +Records individual access events for audit purposes. Deduplicated by `(config_id, external_user_id, scraper_id)` — on conflict, `count` is incremented and `created_at`, `mfa`, and `properties` are updated only if the new record has a more recent `created_at`. + +### Input Shorthands + +| Shorthand | Resolves To | Description | +|-----------|-------------|-------------| +| `user` | `external_user_aliases` | Matched against user aliases | +| `config_id` (non-UUID) | `external_config_id.external_id` | External ID for config lookup | +| `config_type` / `type` | `external_config_id.config_type` | Config type for external ID lookup | diff --git a/mission-control/docs/reference/config-db/scrape-result.md b/mission-control/docs/reference/config-db/scrape-result.md index 72df3356..ec74d9e7 100644 --- a/mission-control/docs/reference/config-db/scrape-result.md +++ b/mission-control/docs/reference/config-db/scrape-result.md @@ -22,3 +22,5 @@ sidebar_position: 7 | `analysis` | Analysis result of the config item | `*AnalysisResult` | | `action` | Action related to the config item | `string` | | `properties` | Properties associated with the config item | `types.Properties` | +| `config_access` | Config access records (who has access). Also accepts `access` as a key | `[]`[ExternalConfigAccess](/docs/reference/config-db/config_access#config-access) | +| `access_logs` | Access log entries (who actually accessed). Also accepts `logs` as a key | `[]`[ExternalConfigAccessLog](/docs/reference/config-db/config_access#config-access-log) | diff --git a/mission-control/docs/reference/connections/facet.mdx b/mission-control/docs/reference/connections/facet.mdx new file mode 100644 index 00000000..d8aeb89b --- /dev/null +++ b/mission-control/docs/reference/connections/facet.mdx @@ -0,0 +1,39 @@ +--- +title: Facet +description: Configure remote HTML/PDF rendering services for report generation +sidebar_custom_props: + icon: file-pdf +--- + +Facet connections configure access to remote HTML/PDF rendering services used by the [Report Action](/docs/guide/playbooks/actions/report). + +## Used By + +- [Report Action](/docs/guide/playbooks/actions/report) — Generate HTML and PDF reports from Views or config catalog queries + +## Schema + +| Field | Description | Scheme | Required | +|----------------|----------------------------------------------------------------------|------------------------------------------------|----------| +| `url` | URL of the Facet rendering service | string | `true` | +| `token` | Authentication token for the rendering service | _EnvVar_ | | +| `timestampUrl` | RFC 3161 timestamp authority URL for cryptographic PDF signing | string | | + +## Example + +```yaml title="facet-connection.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Connection +metadata: + name: renderer + namespace: default +spec: + facet: + url: https://facet.example.com + token: + valueFrom: + secretKeyRef: + name: facet-credentials + key: token + timestampUrl: https://freetsa.org/tsr +``` diff --git a/mission-control/docs/reference/connections/index.mdx b/mission-control/docs/reference/connections/index.mdx index 80fcfb0f..b904b98e 100644 --- a/mission-control/docs/reference/connections/index.mdx +++ b/mission-control/docs/reference/connections/index.mdx @@ -27,7 +27,7 @@ Connections provide a secure, reusable way to authenticate against external syst | **File Storage** | [SFTP](./sftp), [SMB](./smb) | | **AI Providers** | [OpenAI](./openai), [Anthropic](./anthropic), [Ollama](./ollama) | | **Notifications** | [Slack](./slack), [Discord](./discord), [Telegram](./telegram), [SMTP](./smtp), [Ntfy](./ntfy), [Pushbullet](./pushbullet), [Pushover](./pushover) | -| **Generic** | [HTTP](./http) | +| **Generic** | [HTTP](./http), [Facet](./facet) | ## Creating Connections diff --git a/mission-control/docs/reference/playbooks/events.md b/mission-control/docs/reference/playbooks/events.md index e4133b20..a1e71927 100644 --- a/mission-control/docs/reference/playbooks/events.md +++ b/mission-control/docs/reference/playbooks/events.md @@ -73,6 +73,48 @@ spec: message: 'Description: {{.component.description}}' ``` +## Schedule + +Playbooks can be triggered on a recurring cron-based schedule using the `on.schedule` trigger. + +| Field | Description | Scheme | Required | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | -------- | +| `schedule` | Cron expression. Supports standard cron (`0 9 * * MON`), `@every ` (e.g. `@every 1h`), and aliases (`@hourly`, `@daily`, `@weekly`, `@monthly`). Prefix with `CRON_TZ=` to set timezone. Evaluated in UTC by default. | `string` | `true` | +| `parameters` | Default parameters passed to each scheduled run (supports template expressions evaluated at run time) | `map[string]string` | | + +```yaml title="scheduled-every-hour.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: hourly-cleanup +spec: + on: + schedule: + - schedule: "@every 1h" + actions: + - name: cleanup + exec: + script: /opt/scripts/cleanup.sh +``` + +```yaml title="scheduled-cron-with-timezone.yaml" +apiVersion: mission-control.flanksource.com/v1 +kind: Playbook +metadata: + name: daily-morning-report +spec: + on: + schedule: + - schedule: "CRON_TZ=America/New_York 0 9 * * MON-FRI" + parameters: + environment: production + actions: + - name: generate-report + report: + view: default/daily-summary + format: pdf +``` + ## Config Config events relate to activities on config items. diff --git a/mission-control/docs/system-properties.md b/mission-control/docs/system-properties.md index 503bb348..6205c2e8 100644 --- a/mission-control/docs/system-properties.md +++ b/mission-control/docs/system-properties.md @@ -1,5 +1,127 @@ -# Log Levels +# System Properties -# Debug & Trace +Mission Control uses runtime properties for behavior that should be adjustable +without changing code. Properties are string key/value pairs. They are separate +from resource `properties` fields such as connection settings, topology display +properties, or playbook parameter UI hints. -Debug and trace settings are different to debug and trace log levels. Log levels are used for general service output - The log volume scales with functionality, not with data. Debug and trace on the otherhand are defined on individual objects e.g. Canaries, Scrapers, Playbooks etc and scale with data volumes. Resource level debug/trace usually have the resource name/id as a prefix +## Setting Properties + +### Helm + +For Kubernetes installs, set properties under the chart `properties` map: + +```yaml +properties: + logs.disable: "true" + ai.disable: "true" + dashboard.default.view: mission-control-dashboard +``` + +### Properties File + +Mission Control loads `mission-control.properties` when present: + +```properties +log.level=debug +query.log=true +topology.query.timeout=45s +``` + +### Database + +Database-backed properties are stored in the `properties` table and are visible +from Settings > Feature Flags when the UI has permission to read that table. + +```sql +INSERT INTO properties (name, value) +VALUES ('logs.disable', 'true') +ON CONFLICT (name) DO UPDATE SET value = excluded.value; +``` + +### CLI + +Components that expose the common property flags accept repeated `-P` values: + +```sh +mission-control -P log.level=debug -P query.log=true +``` + +## UI Feature Flags + +The UI fetches `/properties` and treats feature flags as properties named +`.disable`. A feature is disabled only when the value is exactly the +string `true`. + +| Property | Effect | +| --- | --- | +| `topology.disable` | Hide or disable topology UI surfaces. | +| `health.disable` | Hide or disable health UI surfaces. | +| `incidents.disable` | Hide or disable incident UI surfaces. | +| `config.disable` | Hide or disable config UI surfaces. | +| `logs.disable` | Hide or disable log UI surfaces. | +| `playbooks.disable` | Hide or disable playbook UI surfaces. | +| `applications.disable` | Hide or disable application UI surfaces. | +| `views.disable` | Hide or disable custom view UI surfaces. | +| `ai.disable` | Hide or disable AI actions and prompts in UI surfaces that check this flag. | +| `agents.disable` | Hide or disable agent UI surfaces. | +| `settings.connections.disable` | Hide or disable connection settings. | +| `settings.users.disable` | Hide or disable user settings. | +| `settings.teams.disable` | Hide or disable team settings. | +| `settings.rules.disable` | Hide or disable rules settings. | +| `settings.config_scraper.disable` | Hide or disable config scraper settings. Also disabled when `config.disable=true`. | +| `settings.topology.disable` | Hide or disable topology settings. Also disabled when `topology.disable=true`. | +| `settings.health.disable` | Hide or disable health settings. Also disabled when `health.disable=true`. | +| `settings.job_history.disable` | Hide or disable job history settings. Also disabled when `health.disable=true`. | +| `settings.feature_flags.disable` | Hide or disable the feature flags settings page. | +| `settings.logging_backends.disable` | Hide or disable logging backend settings. | +| `settings.event_queue_status.disable` | Hide or disable event queue status settings. | +| `settings.organization_profile.disable` | Hide or disable organization profile settings. | +| `settings.notifications.disable` | Hide or disable notification settings. | +| `settings.playbooks.disable` | Hide or disable playbook settings. | +| `settings.integrations.disable` | Hide or disable integration settings. | +| `settings.permissions.disable` | Hide or disable permission settings. | +| `settings.artifacts.disable` | Hide or disable artifact settings. | + +Rows with `source=local` are shown read-only in the Feature Flags page. + +## Global UI Snippets + +`flanksource.ui.snippets` is a local UI property used to inject a global browser +snippet. The value must be a JavaScript function expression. The UI executes it +once after the authenticated user is available and passes `{ user, organization }`. + +```js +({ user, organization }) => { + window.analytics?.identify(user?.id, { + email: user?.email, + organization: organization?.name + }); +} +``` + +Only `flanksource.ui.snippets` properties with `source=local` are executed. +DB-backed rows with this name are visible to the feature-flag API but ignored by +the snippet hook. + +## Dashboard Properties + +| Property | Effect | +| --- | --- | +| `dashboard.default.view` | Backend `/api/dashboard` view selector. Accepts `namespace/name` or name and defaults to `mission-control-dashboard`. | +| `defaults.dashboard_view` | UI sidebar dashboard navigation selector. Accepts view UUID, `namespace/name`, or name. | + +## Proxy Property + +| Property | Effect | +| --- | --- | +| `proxy.disable` | In Clerk auth mode, overrides the organization's `direct` metadata. When true, the UI bypasses the proxy and points API clients at the organization's `backend_url`. | + +## Debug And Trace + +Debug and trace settings are different from debug and trace log levels. Log +levels are used for general service output; the log volume scales with +functionality rather than data volume. Debug and trace are defined on individual +objects such as canaries, scrapers, and playbooks, where volume scales with the +number of resources. Resource-level debug or trace usually uses a property with +the resource name or ID as a prefix. diff --git a/mission-control/sidebars.js b/mission-control/sidebars.js index 9e3aae3d..9d68dde5 100644 --- a/mission-control/sidebars.js +++ b/mission-control/sidebars.js @@ -48,6 +48,11 @@ const sidebars = { type: 'doc', id: 'how-it-works', }, + { + type: 'doc', + id: 'changes', + label: "What's New", + }, { type: 'category', diff --git a/mission-control/static/img/views.svg b/mission-control/static/img/views.svg index e01c749e..c08bc03a 100644 --- a/mission-control/static/img/views.svg +++ b/mission-control/static/img/views.svg @@ -1,4 +1,112 @@ - - -
Views
Views
Queries
Queries
Table
Table
Panels
Panels
Panels
Panels
Panels
Panels
Catalogs
Catalogs
Events
Events
      Prometheus
      Prometheus \ No newline at end of file +<\!-- Do not edit this file with editors other than draw.io --> +<\!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> + + + + + <\!-- Data source boxes --> + <\!-- Configs --> + + Configs + + <\!-- Changes --> + + Changes + + <\!-- Prometheus --> + + + Prometheus + + <\!-- SQL --> + + SQL + + <\!-- HTTP --> + + HTTP + + <\!-- Views (viewTableSelector) --> + + Views + (viewTableSelector) + + <\!-- Arrows from sources to Queries (all route through y=120 then to Queries center x=430) --> + <\!-- Configs -> Queries --> + + + + <\!-- Changes -> Queries --> + + + <\!-- Prometheus -> Queries --> + + + <\!-- SQL -> Queries --> + + + <\!-- HTTP -> Queries --> + + + <\!-- Views -> Queries --> + + + <\!-- Queries box --> + + Queries + + <\!-- Variables label --> + $(var.key) injected here + + + <\!-- Queries -> SQLite --> + + + + <\!-- SQLite box --> + + In-memory SQLite + (join / merge) + + <\!-- merge label --> + optional merge SQL + + + <\!-- SQLite -> PostgreSQL --> + + + + <\!-- PostgreSQL box --> + + PostgreSQL Table + (view_<namespace>_<name>) + + <\!-- PostgreSQL -> Panels (left branch) --> + + + + <\!-- PostgreSQL -> Table (center branch) --> + + + + <\!-- PostgreSQL -> Report (right branch) --> + + + + <\!-- Panels stacked (3 overlapping boxes) --> + + + + Panels + + <\!-- Table output box --> + + Table + + <\!-- Report Action box --> + + Report + Action + + diff --git a/modules/canary-checker b/modules/canary-checker index c7de1457..74069099 160000 --- a/modules/canary-checker +++ b/modules/canary-checker @@ -1 +1 @@ -Subproject commit c7de1457d27d1d932bfc1b010ceb90acd943ed02 +Subproject commit 740690992d7cc0886c93487361952547cd42645e diff --git a/modules/config-db b/modules/config-db index 57489d95..bf01aac4 160000 --- a/modules/config-db +++ b/modules/config-db @@ -1 +1 @@ -Subproject commit 57489d95c2952521e16c2914a167777d29299a2b +Subproject commit bf01aac48a714b3434d06f8957f5943b82535c1d diff --git a/modules/duty b/modules/duty index 69c8f7a5..cef196ba 160000 --- a/modules/duty +++ b/modules/duty @@ -1 +1 @@ -Subproject commit 69c8f7a5aa1545d3959c2182412da88d2be8dd63 +Subproject commit cef196baf2b5c3a45cef3fa2b46926d20db9e6fe diff --git a/modules/mission-control b/modules/mission-control index 7d2db2b3..4b73ab99 160000 --- a/modules/mission-control +++ b/modules/mission-control @@ -1 +1 @@ -Subproject commit 7d2db2b3de741dbfa7dfc9d534218112011f0cab +Subproject commit 4b73ab99e612252d6ebbc9fba401b11c3724e23e diff --git a/modules/mission-control-chart b/modules/mission-control-chart index 5c0862ce..5dcaea26 160000 --- a/modules/mission-control-chart +++ b/modules/mission-control-chart @@ -1 +1 @@ -Subproject commit 5c0862ce262114d9f3dd8900407ff8df643fcc27 +Subproject commit 5dcaea266a16b37730ca18e92801b6c33ec79f0b diff --git a/modules/mission-control-registry b/modules/mission-control-registry index 594d6866..1805bb73 160000 --- a/modules/mission-control-registry +++ b/modules/mission-control-registry @@ -1 +1 @@ -Subproject commit 594d6866a3d34e915da4b69972122a98b845532e +Subproject commit 1805bb73d3aa00b107756993c2fea9dcef5745c1 diff --git a/scripts/render-for-context7/go.mod b/scripts/render-for-context7/go.mod deleted file mode 100644 index 647121a9..00000000 --- a/scripts/render-for-context7/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/flanksource/docs-fix/scripts/render-for-context7 - -go 1.21 \ No newline at end of file diff --git a/scripts/render-for-context7/main.go b/scripts/render-for-context7/main.go deleted file mode 100644 index c04af6eb..00000000 --- a/scripts/render-for-context7/main.go +++ /dev/null @@ -1,388 +0,0 @@ -package main - -import ( - "fmt" - "log" - "os" - "path/filepath" - "regexp" - "strings" -) - -type ComponentReplacement struct { - Pattern *regexp.Regexp - Replacement func(matches []string) string -} - -var componentReplacements = []ComponentReplacement{ - // Remove imports - { - Pattern: regexp.MustCompile(`(?m)^import\s+.*$`), - Replacement: func(matches []string) string { return "" }, - }, - // Remove exports - { - Pattern: regexp.MustCompile(`(?m)^export\s+.*$`), - Replacement: func(matches []string) string { return "" }, - }, - // Fields component - { - Pattern: regexp.MustCompile(``), - Replacement: func(matches []string) string { - if len(matches) > 1 { - return fmt.Sprintf("\n### Fields\n\n[Component: Fields connection=\"%s\"]\n", matches[1]) - } - return "\n### Fields\n\n[Component: Fields]\n" - }, - }, - // Step component - handle multiline - { - Pattern: regexp.MustCompile(`(?s)]*>(.*?)`), - Replacement: func(matches []string) string { - if len(matches) > 2 { - content := strings.TrimSpace(matches[3]) - return fmt.Sprintf("\n**Step %s**: %s\n%s", matches[1], matches[2], content) - } - return "\n**Step**: \n" - }, - }, - // CommonLink component - { - Pattern: regexp.MustCompile(`([^<]*)`), - Replacement: func(matches []string) string { - if len(matches) > 2 { - return fmt.Sprintf("[%s](%s)", matches[2], matches[1]) - } - return "[Link](#)" - }, - }, - // Fields component without attributes - { - Pattern: regexp.MustCompile(``), - Replacement: func(matches []string) string { return "\n### Fields\n\n[Component: Fields]\n" }, - }, - // Tabs component - { - Pattern: regexp.MustCompile(`]*>`), - Replacement: func(matches []string) string { return "\n### Tabs\n" }, - }, - { - Pattern: regexp.MustCompile(``), - Replacement: func(matches []string) string { return "" }, - }, - // TabItem component - { - Pattern: regexp.MustCompile(`]*>`), - Replacement: func(matches []string) string { - if len(matches) > 2 { - return fmt.Sprintf("\n#### %s\n", matches[2]) - } - return "\n#### Tab\n" - }, - }, - { - Pattern: regexp.MustCompile(``), - Replacement: func(matches []string) string { return "" }, - }, - // Simple components - handle multiline - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Screenshot]" }, - }, - { - Pattern: regexp.MustCompile(``), - Replacement: func(matches []string) string { return fmt.Sprintf("[Icon: %s]", matches[1]) }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Health Check Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Lookup Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Action Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Scraper Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Custom Scraper Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Helm Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Config Transform Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Asciinema Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Card Component]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Cards Container]" }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Terminal Output]" }, - }, - { - Pattern: regexp.MustCompile(`]*>([^<]*)`), - Replacement: func(matches []string) string { return matches[1] }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Menu Component]" }, - }, - { - Pattern: regexp.MustCompile(``), - Replacement: func(matches []string) string { return fmt.Sprintf("[Tag: %s]", matches[1]) }, - }, - { - Pattern: regexp.MustCompile(`([^<]*)`), - Replacement: func(matches []string) string { return fmt.Sprintf("**%s**", matches[1]) }, - }, - { - Pattern: regexp.MustCompile(`(?s)]*>.*?`), - Replacement: func(matches []string) string { return "[Advanced Section]" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "[Commercial Feature]" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "[Standard Feature]" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "[Enterprise Feature]" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "[Skip in OSS]" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "[Skip in Commercial]" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "✅" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "❌" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "⚠️" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "[Full Image]" }, - }, - { - Pattern: regexp.MustCompile(`]*/?>`), - Replacement: func(matches []string) string { return "[Custom Field]" }, - }, - { - Pattern: regexp.MustCompile(``), - Replacement: func(matches []string) string { return fmt.Sprintf("[Icon: %s]", matches[1]) }, - }, -} - -// Process code imports -func processCodeImports(content string, fileDir string, rootDir string) string { - codeBlockPattern := regexp.MustCompile("(?s)```([a-z]*)(.*?)```") - - return codeBlockPattern.ReplaceAllStringFunc(content, func(match string) string { - // Extract language and metadata - parts := codeBlockPattern.FindStringSubmatch(match) - if len(parts) < 3 { - return match - } - - lang := parts[1] - meta := parts[2] - - // Check for file= in metadata - filePattern := regexp.MustCompile(`file=["']?([^"'\s]+)["']?`) - fileMatches := filePattern.FindStringSubmatch(meta) - if len(fileMatches) < 2 { - return match - } - - filePath := fileMatches[1] - - // Replace with actual root directory - filePath = strings.ReplaceAll(filePath, "", rootDir) - - // Resolve relative paths - if !filepath.IsAbs(filePath) { - filePath = filepath.Join(fileDir, filePath) - } - - // Read the file - fileContent, err := os.ReadFile(filePath) - if err != nil { - return fmt.Sprintf("```%s\n# Failed to import: %s\n# Error: %s\n```", lang, fileMatches[1], err.Error()) - } - - // Remove file= from metadata - cleanMeta := filePattern.ReplaceAllString(meta, "") - cleanMeta = strings.TrimSpace(cleanMeta) - - return fmt.Sprintf("```%s%s\n%s\n```", lang, cleanMeta, string(fileContent)) - }) -} - -// Process a single file -func processFile(inputPath string, outputPath string, rootDir string) error { - // Read the file - content, err := os.ReadFile(inputPath) - if err != nil { - return fmt.Errorf("failed to read file: %w", err) - } - - // Convert to string - contentStr := string(content) - - // Extract frontmatter - frontmatter := "" - mdxContent := contentStr - - if strings.HasPrefix(contentStr, "---\n") { - parts := strings.SplitN(contentStr[4:], "\n---\n", 2) - if len(parts) == 2 { - frontmatter = "---\n" + parts[0] + "\n---\n\n" - mdxContent = parts[1] - } - } - - // Process code imports - fileDir := filepath.Dir(inputPath) - mdxContent = processCodeImports(mdxContent, fileDir, rootDir) - - // Apply component replacements - for _, replacement := range componentReplacements { - mdxContent = replacement.Pattern.ReplaceAllStringFunc(mdxContent, func(match string) string { - matches := replacement.Pattern.FindStringSubmatch(match) - return replacement.Replacement(matches) - }) - } - - // Combine frontmatter and content - finalContent := frontmatter + mdxContent - - // Ensure output directory exists - outputDir := filepath.Dir(outputPath) - if err := os.MkdirAll(outputDir, 0755); err != nil { - return fmt.Errorf("failed to create output directory: %w", err) - } - - // Write the processed file - if err := os.WriteFile(outputPath, []byte(finalContent), 0644); err != nil { - return fmt.Errorf("failed to write output file: %w", err) - } - - return nil -} - -// Find all markdown files -func findMarkdownFiles(root string) ([]string, error) { - var files []string - - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - // Skip directories - if info.IsDir() { - // Skip node_modules and hidden directories - if strings.HasPrefix(info.Name(), ".") || info.Name() == "node_modules" { - return filepath.SkipDir - } - return nil - } - - // Check for markdown files - ext := filepath.Ext(path) - if ext == ".md" || ext == ".mdx" { - // Skip files starting with underscore - if !strings.HasPrefix(info.Name(), "_") { - files = append(files, path) - } - } - - return nil - }) - - return files, err -} - -func main() { - // Get current working directory (should be the script directory) - scriptDir, err := os.Getwd() - if err != nil { - log.Fatal("Failed to get current directory:", err) - } - - // The actual project directory is passed via PWD environment variable by npm - projectDir := os.Getenv("INIT_CWD") - if projectDir == "" { - // Fallback: assume we're being run from mission-control or canary-checker - projectDir = filepath.Dir(filepath.Dir(scriptDir)) - } - - // Configuration - docsDir := filepath.Join(projectDir, "docs") - outputDir := filepath.Join(projectDir, "rendered-docs") - rootDir := filepath.Dir(projectDir) - - fmt.Printf("Rendering documentation from %s to %s...\n", docsDir, outputDir) - fmt.Printf("Root directory for imports: %s\n", rootDir) - - // Find all markdown files - files, err := findMarkdownFiles(docsDir) - if err != nil { - log.Fatal("Failed to find files:", err) - } - - fmt.Printf("Found %d files to process\n", len(files)) - - // Process each file - successCount := 0 - for _, inputPath := range files { - // Calculate output path - relPath, _ := filepath.Rel(docsDir, inputPath) - outputPath := filepath.Join(outputDir, relPath) - - // Change .mdx to .md - if filepath.Ext(outputPath) == ".mdx" { - outputPath = strings.TrimSuffix(outputPath, ".mdx") + ".md" - } - - // Process the file - if err := processFile(inputPath, outputPath, rootDir); err != nil { - fmt.Printf("✗ Error processing %s: %v\n", relPath, err) - } else { - fmt.Printf("✓ Processed: %s\n", relPath) - successCount++ - } - } - - fmt.Printf("\nRendering complete! Successfully processed %d/%d files.\n", successCount, len(files)) -} \ No newline at end of file