From 2c451ed8813e2004b285f767139c4bd8f1694f06 Mon Sep 17 00:00:00 2001 From: Yauheni Hlazkou Date: Wed, 8 Jul 2026 19:00:14 +0200 Subject: [PATCH 1/6] feat: add technical lineage MCP tools and tech-lineage skill (Snowflake) --- SKILLS.md | 3 +- pkg/clients/catalog_database_client.go | 21 +++ pkg/clients/technical_lineage_client.go | 23 +++ .../files/collibra/etl-integration/SKILL.md | 2 +- .../etl-integration/references/index.md | 3 + pkg/skills/files/collibra/index/SKILL.md | 13 +- .../files/collibra/jdbc-ingestion/SKILL.md | 5 +- pkg/skills/files/collibra/lineage/SKILL.md | 11 +- .../files/collibra/tech-lineage/SKILL.md | 132 ++++++++++++++++++ .../tech-lineage/references/snowflake.md | 89 ++++++++++++ pkg/tools/edge_run_capability/tool.go | 2 +- pkg/tools/get_registered_database/tool.go | 67 +++++++++ .../get_registered_database/tool_test.go | 122 ++++++++++++++++ pkg/tools/register.go | 4 + pkg/tools/start_technical_lineage/tool.go | 48 +++++++ .../start_technical_lineage/tool_test.go | 71 ++++++++++ 16 files changed, 607 insertions(+), 9 deletions(-) create mode 100644 pkg/clients/technical_lineage_client.go create mode 100644 pkg/skills/files/collibra/tech-lineage/SKILL.md create mode 100644 pkg/skills/files/collibra/tech-lineage/references/snowflake.md create mode 100644 pkg/tools/get_registered_database/tool.go create mode 100644 pkg/tools/get_registered_database/tool_test.go create mode 100644 pkg/tools/start_technical_lineage/tool.go create mode 100644 pkg/tools/start_technical_lineage/tool_test.go diff --git a/SKILLS.md b/SKILLS.md index 2fbc277..95722a3 100644 --- a/SKILLS.md +++ b/SKILLS.md @@ -14,11 +14,12 @@ those tools. Skill content lives in [`pkg/skills/files/collibra/`](pkg/skills/fi |---|---| | `collibra/index` | Navigator — start here when unsure which skill applies | | `collibra/discovery` | Semantic vs keyword search; resolving names to UUIDs | -| `collibra/lineage` | Technical lineage; DGC UUID ↔ lineage entity ID bridge; column-level workaround | +| `collibra/lineage` | Querying existing technical lineage; DGC UUID ↔ lineage entity ID bridge; column-level workaround | | `collibra/asset-create` | `create_asset` workflow; RICH_TEXT Markdown handling; duplicate gating | | `collibra/asset-edit` | `edit_asset` operation types | | `collibra/jdbc-ingestion` | End-to-end jdbc-ingestion setup: Edge connection/capability, database registration, running and polling a sync | | `collibra/etl-integration` | End-to-end ETL sync integrations (Dataplex, Databricks UC, Purview, Sigma, …): connection, capability, generic config (fetched live), schedule/run; per-integration references | +| `collibra/tech-lineage` | Technical lineage harvesting setup: prerequisites (Database asset + jdbc-ingestion), Technical Lineage capability, `start_technical_lineage` trigger; per-source references (Snowflake) | Each skill is one `SKILL.md` per directory, with frontmatter (`description`, `related`) and an optional `references/` directory for bundled reference documents. diff --git a/pkg/clients/catalog_database_client.go b/pkg/clients/catalog_database_client.go index 6a998a5..0768ef7 100644 --- a/pkg/clients/catalog_database_client.go +++ b/pkg/clients/catalog_database_client.go @@ -122,6 +122,27 @@ func RegisterDatabase(ctx context.Context, client *http.Client, request AddDatab return &database, nil } +// GetDatabase fetches a registered Database asset via GET /databases/{databaseId}. +func GetDatabase(ctx context.Context, client *http.Client, databaseID string) (*Database, error) { + endpoint := catalogDatabaseBasePath + "/databases/" + databaseID + var database Database + if err := doCatalogDatabaseGet(ctx, client, endpoint, &database); err != nil { + return nil, err + } + return &database, nil +} + +// GetDatabaseConnection fetches a database connection via +// GET /databaseConnections/{databaseConnectionId}. +func GetDatabaseConnection(ctx context.Context, client *http.Client, databaseConnectionID string) (*DatabaseConnection, error) { + endpoint := catalogDatabaseBasePath + "/databaseConnections/" + databaseConnectionID + var connection DatabaseConnection + if err := doCatalogDatabaseGet(ctx, client, endpoint, &connection); err != nil { + return nil, err + } + return &connection, nil +} + // RefreshSchemaConnections asynchronously refreshes the schema connections known for a // database connection via POST /schemaConnections/refresh?databaseConnectionId=... func RefreshSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) error { diff --git a/pkg/clients/technical_lineage_client.go b/pkg/clients/technical_lineage_client.go new file mode 100644 index 0000000..50f7149 --- /dev/null +++ b/pkg/clients/technical_lineage_client.go @@ -0,0 +1,23 @@ +package clients + +import ( + "context" + "fmt" + "net/http" +) + +const technicalLineageBasePath = "/rest/catalog/1.0/technicalLineage" + +// StartTechnicalLineageHarvest triggers the technical lineage harvest for a registered +// Database asset via POST /harvester/{assetId}. The endpoint accepts the request with +// 202 and an empty body — the DGC job it spawns is not returned and must be located +// through the jobs API afterwards. +func StartTechnicalLineageHarvest(ctx context.Context, client *http.Client, assetID string) error { + endpoint := technicalLineageBasePath + "/harvester/" + assetID + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} diff --git a/pkg/skills/files/collibra/etl-integration/SKILL.md b/pkg/skills/files/collibra/etl-integration/SKILL.md index 71d17b3..e32cf68 100644 --- a/pkg/skills/files/collibra/etl-integration/SKILL.md +++ b/pkg/skills/files/collibra/etl-integration/SKILL.md @@ -1,6 +1,6 @@ --- description: Set up, configure, schedule, and run Edge ETL integrations (metadata sync capabilities like Dataplex, Databricks Unity Catalog, Purview, Sigma) end to end. -related: collibra/jdbc-ingestion, collibra/discovery +related: collibra/jdbc-ingestion, collibra/discovery, collibra/tech-lineage --- # ETL integrations — connect, configure, schedule, run diff --git a/pkg/skills/files/collibra/etl-integration/references/index.md b/pkg/skills/files/collibra/etl-integration/references/index.md index fa1d336..3c3aee8 100644 --- a/pkg/skills/files/collibra/etl-integration/references/index.md +++ b/pkg/skills/files/collibra/etl-integration/references/index.md @@ -34,6 +34,9 @@ conceptual, not field-level. through the `catalog_etl_*` tools — see "Not controllable via chip (yet)" in `SKILL.md`. Notes: +- **JDBC technical lineage capabilities (e.g. "Technical Lineage for Snowflake") are not + `catalog_etl_*` integrations** — they are triggered per Database asset via + `start_technical_lineage`. See `collibra/tech-lineage`. - ThoughtSpot's manifest uses a generated typeId; confirm the live value with `edge_list_capability_types` if `edge_create_capability` rejects the id above. - SAP capabilities (AI Core, Datasphere, Data Products, S/4HANA) are out of scope for this diff --git a/pkg/skills/files/collibra/index/SKILL.md b/pkg/skills/files/collibra/index/SKILL.md index c5719b3..a4b9285 100644 --- a/pkg/skills/files/collibra/index/SKILL.md +++ b/pkg/skills/files/collibra/index/SKILL.md @@ -1,6 +1,6 @@ --- description: Navigator for chip's Collibra skills. Start here when unsure which skill applies. -related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/jdbc-ingestion, collibra/etl-integration +related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/jdbc-ingestion, collibra/etl-integration, collibra/tech-lineage --- # Collibra skills — navigator @@ -14,13 +14,22 @@ must be bridged to another, and which permissions are required. | Task | Skill | |---|---| | Find data about a topic by meaning (semantic) vs by exact name (keyword) | `collibra/discovery` | -| Trace upstream sources, downstream consumers, or impact of a change | `collibra/lineage` | +| Trace upstream sources, downstream consumers, or impact of a change in **existing** lineage (query only) | `collibra/lineage` | | Create any new asset (Business Term, Table, Column, KPI, …) | `collibra/asset-create` | | Modify an existing asset's attributes, relations, tags, status, or owners | `collibra/asset-edit` | | Register a table (and its dimension tables) as a Collibra Data Product with ports | `collibra/data-product-create` | | Generate governed YAML context (semantic blueprints, metric definitions) for an asset | `collibra/context` | | Set up an Edge JDBC connection/capability and run jdbc-ingestion | `collibra/jdbc-ingestion` | | Set up any metadata-sync ETL integration (Dataplex, Databricks UC, Purview, Sigma, …) — connection, capability, generic config, schedule/run | `collibra/etl-integration` | +| **Create, set up, enable, or run technical lineage** for a registered database (Snowflake, …) | `collibra/tech-lineage` | + +**"Technical lineage" is ambiguous — route by verb, and don't ask the user which skill +to use.** *Querying* what already exists (trace, upstream, downstream, impact, "where +does this come from") → `collibra/lineage`. *Producing* it (create, set up, configure, +enable, run, harvest, ingest lineage for a source) → `collibra/tech-lineage`. If the +request genuinely doesn't say (e.g. just "technical lineage for Snowflake"), ask whether +they want to explore existing lineage or set up harvesting — never present the skill +names themselves as the choice. If a task is a single tool call with no chaining (e.g. `get_asset_details` by UUID, `list_asset_types`, `pull_data_contract_manifest`), no skill is needed — the tool's own diff --git a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md index ca5e552..40873a2 100644 --- a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md +++ b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md @@ -1,6 +1,6 @@ --- description: Set up an Edge JDBC connection and capability, register a database, and run jdbc-ingestion end to end. -related: collibra/discovery +related: collibra/discovery, collibra/tech-lineage --- # JDBC ingestion — connect, register, and run @@ -75,6 +75,9 @@ which ingestion type it should be** — don't assume JDBC just because this skil 202/success from `start_ingestion` only means the job was accepted, not that ingestion finished — always poll to confirm completion. +Once ingestion has completed, **technical lineage** can be layered on top of the +registered database — see `collibra/tech-lineage`. + ## Two job-id spaces — do not cross them - **Edge-site jobs** (from `test_connection`) → poll with **`edge_get_job_status`**. diff --git a/pkg/skills/files/collibra/lineage/SKILL.md b/pkg/skills/files/collibra/lineage/SKILL.md index e1d7d49..66005b4 100644 --- a/pkg/skills/files/collibra/lineage/SKILL.md +++ b/pkg/skills/files/collibra/lineage/SKILL.md @@ -1,9 +1,14 @@ --- -description: Trace upstream sources and downstream consumers in Collibra's technical lineage graph. Covers the DGC UUID → lineage entity ID bridge. -related: collibra/discovery, collibra/index +description: Query the existing technical lineage graph — trace upstream sources and downstream consumers. Covers the DGC UUID → lineage entity ID bridge. Read-only — for setting up, creating, or running technical lineage harvesting, load collibra/tech-lineage instead. +related: collibra/discovery, collibra/index, collibra/tech-lineage --- -# Technical lineage +# Technical lineage — querying the graph + +This skill only covers **querying** existing lineage. If the user wants to *set up*, +*create*, *configure*, *enable*, or *run* technical lineage (harvest lineage from a data +source like Snowflake), **this is the wrong skill — load `collibra/tech-lineage`** +before doing anything else. Technical lineage maps the physical data flow — including unregistered assets, temporary tables, and source code — across systems. This is broader than business lineage (which only diff --git a/pkg/skills/files/collibra/tech-lineage/SKILL.md b/pkg/skills/files/collibra/tech-lineage/SKILL.md new file mode 100644 index 0000000..ab655ff --- /dev/null +++ b/pkg/skills/files/collibra/tech-lineage/SKILL.md @@ -0,0 +1,132 @@ +--- +description: Set up, create, configure, and run technical lineage harvesting for a registered database (Snowflake, …) — resolve the connection, create the Technical Lineage capability, trigger and track the harvest. Use for any "create/set up/enable/run technical lineage" request; for querying existing lineage, load collibra/lineage instead. +related: collibra/jdbc-ingestion, collibra/etl-integration, collibra/lineage, collibra/discovery +--- + +# Technical lineage — prerequisites, capability, harvest + +Technical lineage extracts data-flow lineage (views, procedures, query history) from a +data source and loads it into Collibra's lineage graph. On Edge it is a **capability** +(display name "Technical Lineage for ") that reads the source through the same +**Generic JDBC connection** used for jdbc-ingestion, and is triggered through a dedicated +catalog endpoint keyed by the **Database asset** — not by the capability. + +This skill is about *producing* technical lineage. For *querying* the lineage graph +(upstream/downstream, transformations), see `collibra/lineage`. Some sources have their +own lineage sync capabilities managed as ETL integrations instead (Databricks, GCP/Dataplex) +— those go through `collibra/etl-integration`, not this skill. + +## Prerequisites — check these first, before any setup + +Technical lineage builds on top of a completed jdbc-ingestion: + +1. A **Database asset** must exist for the data source, registered via + `configure_database` (`collibra/jdbc-ingestion`). Its UUID is the key that triggers + the harvest. +2. **JDBC ingestion must have run** (`start_ingestion`) — harvested lineage stitches to + the ingested Schema/Table/Column assets; without them the lineage has nothing to + attach to. + +Verify before proceeding: resolve the Database asset with `search_asset_keyword` +(filter by asset type `"Database"`; `collibra/discovery` has the name→UUID rules). If no +Database asset exists, or ingestion never ran: **stop and tell the user** that a +registered database and a completed JDBC ingestion are prerequisites for technical +lineage, and offer to set them up first via `collibra/jdbc-ingestion`. Do not try to +create the Database asset any other way. + +## The full sequence + +1. **Resolve the Database asset** (see Prerequisites) → keep its UUID; it is both the + input of step 2 and the `assetId` for step 5. +2. **`get_registered_database`** with that UUID → the **Edge connection** backing the + ingestion and its **`edgeSiteId`**. Both are derived, not chosen: the connection id + feeds the capability's `connection` parameter, and the capability **must be created + on that same edge site**. A failure here means the asset is not a registered + database — treat it as a missing prerequisite (see above). Only if this tool is + unavailable, fall back to `edge_list_sites` + `edge_find_connections` and confirm + the picks with the user. +3. **`edge_list_capability_types`** (the `edgeSiteId` from step 2; pass a `query`, e.g. + `"snowflake"`) → find the **Technical Lineage** capability type and read its + parameter manifest. **Several capability types can match the data source name — only + the technical-lineage one is correct.** See "Exactly one right capability type". +4. **`edge_create_capability`** → `typeId` from step 3, `edgeSiteId` and + `parameters.connection` from step 2, remaining `parameters` from the manifest — ask + the user for the values and confirm the full set first. See "Capability parameters". + Skip if a Technical Lineage capability for this source already exists + (`edge_find_capabilities`). +5. **`start_technical_lineage`** with the Database asset UUID from step 1 — **not** the + capability id. Success means *submitted*, nothing more. +6. **Track:** the trigger returns no job id. Find the spawned DGC job with **`jobs_find`** + (jobs started at/after the trigger), then poll it with **`get_job_status`** until it + reaches a terminal state. Never poll with `edge_get_job_status` — the Edge-side + harvest is tracked through the DGC job. + +## Exactly one right capability type + +An Edge site lists many capability types, and more than one can contain the data-source +name (e.g. jdbc-ingestion profiles, sync capabilities, *and* technical lineage for +Snowflake). The technical-lineage capability is recognizable by its identity, not by the +name substring alone — for Snowflake it is the type whose manifest displays +**"Technical Lineage for Snowflake"** (see `references/snowflake.md` for the expected +`typeId`). If the expected type is not present on the site, list what +`edge_list_capability_types` returned and ask the user — never fall back to another +`*snowflake*` type. + +In particular, **capability types for SQL files in cloud storage** (lineage harvested +from `.sql` files in an S3/ADLS/GCS bucket) can also carry the dialect name and their +manifests ask for file/storage locations. Those belong to the separate "lineage for +cloud storage" flow — see "Not supported" below. This skill's flow harvests **directly +from the database** over the JDBC connection; there are no files to locate, so never +ask the user where files are stored. + +## Not supported: lineage for cloud storage + +Technical lineage from **SQL files in cloud storage** (S3, ADLS, GCS buckets) is a +different flow with different capability types and parameters, and is **not supported +via chip yet**. If the user's lineage source is SQL files in a bucket rather than a +registered database, tell them it must be configured in the Collibra/Edge UI — do not +attempt it with this skill's tools or pick a cloud-storage capability type. + +## Capability parameters + +Read the parameter names, requiredness, defaults, and options from the **live manifest** +(`edge_list_capability_types` with a `query`) — never from memory; manifests evolve with +the capability version. The per-source reference (`references/snowflake.md`) explains +what the fields *mean* and how to choose values; the manifest stays authoritative for +exact names and defaults. + +To fill them in: ask the user for the required values, propose defaults for the +optional ones, and **confirm the complete parameter set with the user before calling +`edge_create_capability`**. Never ask for database credentials — they live in the Edge +connection, not in capability parameters. + +## Trigger and track — 202 means submitted, not finished + +`start_technical_lineage` wraps a bare POST that returns `202 Accepted` with no body: + +- A success response **only** means DGC accepted the harvest request. +- No job id is returned. Locate the spawned DGC job with `jobs_find` right after the + call (filter to jobs started at/after the trigger) and poll it with `get_job_status` + to a terminal state before reporting the harvest as done. If the job ends in failure, + surface its message/status log to the user. + +## Hard rules + +1. **Verify the prerequisites first** (Database asset + completed ingestion). When + missing, stop and route to `collibra/jdbc-ingestion` — don't improvise. +2. **Read capability parameters from `edge_list_capability_types`,** not memory; use + `references/.md` for meaning, the manifest for names/defaults. +3. **Ask the user for parameter values and confirm the full set** before + `edge_create_capability`. Never ask for credentials. The `connection` value and the + `edgeSiteId` are not user choices — take both from `get_registered_database`; the + capability lives on the same edge site as the connection. +4. **Only `start_technical_lineage` triggers a harvest** — never `edge_run_capability` + and never `catalog_etl_start_job`. Pass the **Database asset** UUID, not the + capability id. +5. **Always locate and poll the DGC job after triggering** (`jobs_find` → + `get_job_status`). A 202/success alone is not a completed harvest. +6. **When several capability types match the source name, never pick one silently** — + match the technical-lineage identity or ask. +7. **Databases only.** This flow harvests from the database over the JDBC connection — + never ask where files are stored. Lineage for SQL files in cloud storage is not + supported via chip; route those users to the Collibra/Edge UI. \ No newline at end of file diff --git a/pkg/skills/files/collibra/tech-lineage/references/snowflake.md b/pkg/skills/files/collibra/tech-lineage/references/snowflake.md new file mode 100644 index 0000000..f1cbe86 --- /dev/null +++ b/pkg/skills/files/collibra/tech-lineage/references/snowflake.md @@ -0,0 +1,89 @@ +# Technical Lineage for Snowflake (`edgeharvester-snowflake`) + +Harvests lineage from Snowflake — view/procedure SQL or `ACCESS_HISTORY` — and loads it +into the Collibra lineage graph, stitched to the assets ingested by jdbc-ingestion. + +- **Capability typeId** (for `edge_create_capability`): **`edgeharvester-snowflake`** — + the manifest displays **"Technical Lineage for Snowflake"**. Confirm the live id with + `edge_list_capability_types(query: "snowflake")`; do not confuse it with other + Snowflake-named capability types on the site — especially not with the SQL-files / + cloud-storage lineage variants, whose manifests ask for file or bucket locations + (that flow is not supported via chip). +- **Connection:** the Generic JDBC connection (`typeId: "Generic"`) created for + jdbc-ingestion — resolve it (and the edge site for the capability) from the Database + asset with `get_registered_database`. Credentials, role, and warehouse live in the + connection — the capability never carries them. +- **Trigger:** `start_technical_lineage(assetId)` with the **Database asset** UUID. + +**Docs:** [Technical lineage for Snowflake](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/Snowflake/to_snowflake.htm) + +## Snowflake access requirements + +The connection's role needs access to `SNOWFLAKE.ACCOUNT_USAGE` views (`TABLES`, +`COLUMNS`, `VIEWS`, `PROCEDURES`, `OBJECT_DEPENDENCIES`, `ACCESS_HISTORY`, +`QUERY_HISTORY`, …) — typically granted with `IMPORTED PRIVILEGES` on the shared +`SNOWFLAKE` database. Without it the harvest queries fail even though `test_connection` +succeeds. + +## Capability parameters — Main Properties + +- `id` (required) — **Source ID**: a unique identifier for this lineage source. It keys + the source on the lineage server: re-running with the same `id` replaces that source's + lineage; other sources can reference it via `dependentSourceIds`. Pick a stable, + descriptive value (e.g. `snowflake-prod`). +- `connection` (required) — the Generic JDBC connection id (from + `get_registered_database`). +- `collibraSystemName` (required) — the System asset name used to stitch harvested + lineage to catalog assets. Must match the system under which jdbc-ingestion registered + the database's assets. +- `snowflakeMode` (required) — how lineage is derived: + - **`SQL`** — parses the SQL of views and stored procedures. Gives lineage for + defined objects; requires `databaseNames`. + - **`SQL-API`** — reads `ACCESS_HISTORY` + `OBJECT_DEPENDENCIES`, i.e. lineage from + what actually ran, including ad-hoc DML. Supports `extraDatabaseDefinitions` and + `snowflakeDays`. +- `databaseNames` (list; required in `SQL` mode) — the databases to harvest. Collect the + names from the user directly; leave the manifest's `databaseNamesJson` file-upload + variant unset (there are no files in this flow). +- `schemaNames` (list, optional) — restrict harvesting to these schemas. +- `snowflakeDays` (default `365`) — how many days of `ACCESS_HISTORY` to read + (`SQL-API` mode). Lower it for faster harvests on busy accounts. +- `extraDatabaseDefinitions` (list, `SQL-API` only) — databases to load metadata for + without harvesting lineage from them (targets of cross-database references). + +## Queries group + +One editable SQL parameter per query type (`columns`, `views`, `procedures`, and in +`SQL-API` mode `object_dependencies`, `access_history`, …), pre-filled with the default +`ACCOUNT_USAGE` queries. Leave the defaults unless the user explicitly needs to override +them (e.g. custom filtering); placeholders like `##DBNAMES##`, `##SCHEMANAMES##`, +`##DAYS##` are substituted at run time. + +## Advanced Properties + +- `processingLevel` — `Load` (harvest only, raw metadata inspectable), `Analyze` + (harvest + analyze), `Sync` (default: also synchronize lineage into the catalog). +- `dependentSourceIds` — Source IDs of other lineage sources this one references, so + cross-source lineage resolves. +- `databaseSystemMapping` — map database names to system names when one capability + covers databases that belong to different systems. +- `deleteRawMetadataAfterProcessing` — remove harvested raw metadata after processing. +- `active` — set to `false` and re-run to **remove** this source's lineage from the + lineage server. +- `techlinAdminConnection` (beta) — optional connection to the lineage (TechLin) server + admin API. Leave unset unless the user asks for it. + +## Gotchas + +- **Stitching needs ingested assets.** Lineage attaches to the Schema/Table/Column + assets created by jdbc-ingestion under `collibraSystemName`; if ingestion never ran + (or ran under a different system name), the harvest succeeds but lineage doesn't link + to catalog assets. +- **The trigger is asset-keyed.** `start_technical_lineage` takes the Database asset + UUID; DGC finds the matching Technical Lineage capability itself. Passing the + capability id fails. +- **`SQL` vs `SQL-API` changes the parameter set** (which query params exist, whether + `databaseNames` is required) — re-read the manifest after the user picks the mode. +- The live manifest from `edge_list_capability_types` is authoritative for exact + parameter names, requiredness, and defaults on the customer's capability version; + this reference explains meaning and choice. \ No newline at end of file diff --git a/pkg/tools/edge_run_capability/tool.go b/pkg/tools/edge_run_capability/tool.go index 2c40162..6163caf 100644 --- a/pkg/tools/edge_run_capability/tool.go +++ b/pkg/tools/edge_run_capability/tool.go @@ -28,7 +28,7 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "edge_run_capability", - Description: "Runs an Edge capability immediately. Returns the job UUID that can be used to track execution via edge_get_job_status.", + Description: "Runs an Edge capability immediately. Returns the job UUID that can be used to track execution via edge_get_job_status. Do not use this for jdbc-ingestion (use start_ingestion) or technical lineage (use start_technical_lineage) — those flows have dedicated DGC triggers that a raw capability run would bypass.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/get_registered_database/tool.go b/pkg/tools/get_registered_database/tool.go new file mode 100644 index 0000000..c4bdb92 --- /dev/null +++ b/pkg/tools/get_registered_database/tool.go @@ -0,0 +1,67 @@ +// Package get_registered_database implements the get_registered_database MCP tool: +// resolves a registered Database asset to the Edge connection (and edge site) behind it. +package get_registered_database + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + AssetID string `json:"assetId" jsonschema:"UUID of the registered Database asset (as created by configure_database)."` +} + +type Output struct { + Database *clients.Database `json:"database,omitempty" jsonschema:"The registered database."` + EdgeConnection *clients.EdgeConnection `json:"edgeConnection,omitempty" jsonschema:"The Edge connection backing the registered database. Its id is the value for a capability's 'connection' parameter; its edgeSiteId is the edge site where dependent capabilities (e.g. Technical Lineage) must be created."` + Success bool `json:"success" jsonschema:"Whether the database was resolved to its Edge connection."` + Error string `json:"error,omitempty" jsonschema:"Error message if resolving failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_registered_database", + Title: "Get Registered Database", + Description: "Resolves a registered Database asset to the Edge connection behind it (database → database connection → Edge connection), returning the connection and its edgeSiteId. Use it to reuse the ingestion's connection when setting up dependent capabilities such as Technical Lineage — the capability must be created on the returned edgeSiteId. Fails if the asset is not a database registered via configure_database.", + Handler: handler(collibraClient), + Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("assetId", input.AssetID); err != nil { + return Output{}, err + } + + database, err := clients.GetDatabase(ctx, collibraClient, input.AssetID) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to get registered database (is this the UUID of a Database asset registered via configure_database?): %s", err.Error())}, nil + } + if database.DatabaseConnectionID == "" { + return Output{Database: database, Success: false, Error: "registered database has no database connection"}, nil + } + + databaseConnection, err := clients.GetDatabaseConnection(ctx, collibraClient, database.DatabaseConnectionID) + if err != nil { + return Output{Database: database, Success: false, Error: fmt.Sprintf("failed to get database connection %s: %s", database.DatabaseConnectionID, err.Error())}, nil + } + if databaseConnection.EdgeConnectionID == "" { + return Output{Database: database, Success: false, Error: "database connection has no Edge connection"}, nil + } + + edgeConnection, err := clients.GetConnection(ctx, collibraClient, databaseConnection.EdgeConnectionID) + if err != nil { + return Output{Database: database, Success: false, Error: fmt.Sprintf("failed to get Edge connection %s: %s", databaseConnection.EdgeConnectionID, err.Error())}, nil + } + + return Output{Database: database, EdgeConnection: edgeConnection, Success: true}, nil + } +} diff --git a/pkg/tools/get_registered_database/tool_test.go b/pkg/tools/get_registered_database/tool_test.go new file mode 100644 index 0000000..b4aa3af --- /dev/null +++ b/pkg/tools/get_registered_database/tool_test.go @@ -0,0 +1,122 @@ +package get_registered_database_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/get_registered_database" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestGetRegisteredDatabase_Success(t *testing.T) { + assetID, _ := uuid.NewUUID() + databaseConnID, _ := uuid.NewUUID() + edgeConnID, _ := uuid.NewUUID() + edgeSiteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /rest/catalogDatabase/v1/databases/"+assetID.String(), + testutil.JsonHandlerOut(func(r *http.Request) (int, clients.Database) { + return http.StatusOK, clients.Database{ID: assetID.String(), Name: "snowflake-prod", DatabaseConnectionID: databaseConnID.String()} + })) + handler.Handle("GET /rest/catalogDatabase/v1/databaseConnections/"+databaseConnID.String(), + testutil.JsonHandlerOut(func(r *http.Request) (int, clients.DatabaseConnection) { + return http.StatusOK, clients.DatabaseConnection{ID: databaseConnID.String(), EdgeConnectionID: edgeConnID.String()} + })) + handler.Handle("GET /edge/api/rest/v2/connections/"+edgeConnID.String(), + testutil.JsonHandlerOut(func(r *http.Request) (int, clients.EdgeConnection) { + return http.StatusOK, clients.EdgeConnection{Id: edgeConnID.String(), Name: "Snowflake JDBC", TypeId: "Generic", EdgeSiteId: edgeSiteID.String()} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{AssetID: assetID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.EdgeConnection.Id != edgeConnID.String() { + t.Fatalf("expected edge connection id %s, got %s", edgeConnID.String(), output.EdgeConnection.Id) + } + if output.EdgeConnection.EdgeSiteId != edgeSiteID.String() { + t.Fatalf("expected edge site id %s, got %s", edgeSiteID.String(), output.EdgeConnection.EdgeSiteId) + } + if output.Database.Name != "snowflake-prod" { + t.Fatalf("expected database name snowflake-prod, got %s", output.Database.Name) + } +} + +func TestGetRegisteredDatabase_InvalidAssetID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{AssetID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid assetId") + } +} + +func TestGetRegisteredDatabase_NotRegistered(t *testing.T) { + assetID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/catalogDatabase/v1/databases/"+assetID.String(), + func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "database not found", http.StatusNotFound) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{AssetID: assetID.String()}) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure for unregistered database") + } + if output.Error == "" { + t.Fatalf("expected error message in output") + } +} + +func TestGetRegisteredDatabase_EdgeConnectionLookupFails(t *testing.T) { + assetID, _ := uuid.NewUUID() + databaseConnID, _ := uuid.NewUUID() + edgeConnID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /rest/catalogDatabase/v1/databases/"+assetID.String(), + testutil.JsonHandlerOut(func(r *http.Request) (int, clients.Database) { + return http.StatusOK, clients.Database{ID: assetID.String(), DatabaseConnectionID: databaseConnID.String()} + })) + handler.Handle("GET /rest/catalogDatabase/v1/databaseConnections/"+databaseConnID.String(), + testutil.JsonHandlerOut(func(r *http.Request) (int, clients.DatabaseConnection) { + return http.StatusOK, clients.DatabaseConnection{ID: databaseConnID.String(), EdgeConnectionID: edgeConnID.String()} + })) + handler.HandleFunc("GET /edge/api/rest/v2/connections/"+edgeConnID.String(), + func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "connection not found", http.StatusNotFound) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{AssetID: assetID.String()}) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when Edge connection lookup fails") + } + if output.Database == nil { + t.Fatalf("expected the resolved database to be returned alongside the error") + } +} diff --git a/pkg/tools/register.go b/pkg/tools/register.go index edf6d62..c822534 100644 --- a/pkg/tools/register.go +++ b/pkg/tools/register.go @@ -55,6 +55,7 @@ import ( "github.com/collibra/chip/pkg/tools/get_lineage_transformation" "github.com/collibra/chip/pkg/tools/get_lineage_upstream" "github.com/collibra/chip/pkg/tools/get_measure_data" + "github.com/collibra/chip/pkg/tools/get_registered_database" "github.com/collibra/chip/pkg/tools/get_table_semantics" "github.com/collibra/chip/pkg/tools/init_data_contract" "github.com/collibra/chip/pkg/tools/jobs_find" @@ -73,6 +74,7 @@ import ( "github.com/collibra/chip/pkg/tools/search_lineage_entities" "github.com/collibra/chip/pkg/tools/search_lineage_transformations" "github.com/collibra/chip/pkg/tools/start_ingestion" + "github.com/collibra/chip/pkg/tools/start_technical_lineage" "github.com/collibra/chip/pkg/tools/test_connection" "github.com/collibra/chip/pkg/tools/upload_file" ) @@ -141,6 +143,8 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, register_database.NewTool(client)) toolRegister(server, toolConfig, configure_database_schemas.NewTool(client)) toolRegister(server, toolConfig, start_ingestion.NewTool(client)) + toolRegister(server, toolConfig, get_registered_database.NewTool(client)) + toolRegister(server, toolConfig, start_technical_lineage.NewTool(client)) // Edge Management tools — read/delete/run operations with no counterpart among the // create/find/test/get-status lifecycle tools above (edge_create_connection and diff --git a/pkg/tools/start_technical_lineage/tool.go b/pkg/tools/start_technical_lineage/tool.go new file mode 100644 index 0000000..6e48164 --- /dev/null +++ b/pkg/tools/start_technical_lineage/tool.go @@ -0,0 +1,48 @@ +// Package start_technical_lineage implements the start_technical_lineage MCP tool: +// triggers the technical lineage harvest for a registered Database asset. +package start_technical_lineage + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + AssetID string `json:"assetId" jsonschema:"UUID of the Database asset to harvest technical lineage for — the asset registered by jdbc-ingestion (configure_database), NOT the capability id. A Technical Lineage capability referencing the database's connection must already exist on the edge site."` +} + +type Output struct { + Success bool `json:"success" jsonschema:"Whether the harvest was submitted. Submission only — the harvest runs asynchronously and no job id is returned; locate the spawned DGC job with jobs_find and poll it with get_job_status."` + Error string `json:"error,omitempty" jsonschema:"Error message if submitting the harvest failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "start_technical_lineage", + Title: "Start Technical Lineage Harvest", + Description: "Triggers the technical lineage harvest for a registered Database asset. The database must already be registered (configure_database) and ingested (start_ingestion), and a Technical Lineage capability must exist on the edge site. This is the only correct trigger for technical lineage — never use edge_run_capability or catalog_etl_start_job for it. A success response only means the harvest was submitted (the endpoint returns no job id): locate the spawned DGC job with jobs_find and poll it with get_job_status (NOT edge_get_job_status — the Edge-side harvest is tracked through the DGC job).", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("assetId", input.AssetID); err != nil { + return Output{}, err + } + + if err := clients.StartTechnicalLineageHarvest(ctx, collibraClient, input.AssetID); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to start technical lineage harvest: %s", err.Error())}, nil + } + + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/start_technical_lineage/tool_test.go b/pkg/tools/start_technical_lineage/tool_test.go new file mode 100644 index 0000000..3885ff1 --- /dev/null +++ b/pkg/tools/start_technical_lineage/tool_test.go @@ -0,0 +1,71 @@ +package start_technical_lineage_test + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/start_technical_lineage" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestStartTechnicalLineage_Success(t *testing.T) { + assetID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("POST /rest/catalog/1.0/technicalLineage/harvester/"+assetID.String(), + func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + if len(body) != 0 { + t.Fatalf("expected empty request body, got: %s", body) + } + w.WriteHeader(http.StatusAccepted) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{AssetID: assetID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestStartTechnicalLineage_InvalidAssetID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{AssetID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid assetId") + } +} + +func TestStartTechnicalLineage_ServerError(t *testing.T) { + assetID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("POST /rest/catalog/1.0/technicalLineage/harvester/"+assetID.String(), + func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "no technical lineage capability found for asset", http.StatusNotFound) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{AssetID: assetID.String()}) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure on server error") + } + if output.Error == "" { + t.Fatalf("expected error message in output") + } +} From 3dfdd6b1994a5fa28260c4e56a74149fe75d5e0d Mon Sep 17 00:00:00 2001 From: Yauheni Hlazkou Date: Thu, 9 Jul 2026 14:14:26 +0200 Subject: [PATCH 2/6] feat: capability parameter guidance for tech-lineage skill (mode-first, unique source id, techlin custom properties) --- .../files/collibra/tech-lineage/SKILL.md | 15 ++++++++ .../tech-lineage/references/snowflake.md | 38 +++++++++++++++---- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/pkg/skills/files/collibra/tech-lineage/SKILL.md b/pkg/skills/files/collibra/tech-lineage/SKILL.md index ab655ff..1fe0663 100644 --- a/pkg/skills/files/collibra/tech-lineage/SKILL.md +++ b/pkg/skills/files/collibra/tech-lineage/SKILL.md @@ -100,6 +100,21 @@ optional ones, and **confirm the complete parameter set with the user before cal `edge_create_capability`**. Never ask for database credentials — they live in the Edge connection, not in capability parameters. +Ordering and rules for the conversation: + +1. **If the source has an ingestion-mode parameter (Snowflake: `snowflakeMode`), settle + it first** — the mode decides which of the remaining parameters exist and which are + required, so asking anything else before it wastes the user's time. Re-check the + manifest against the chosen mode before continuing. +2. **Source ID must be unique.** Propose a value, but never one that collides with an + existing lineage source id (check the `id` parameter of existing lineage + capabilities via `edge_list_capabilities`/`edge_find_capabilities`), with the + `collibraSystemName`, or with a database/schema name. A colliding Source ID silently + overwrites or cross-links another source's lineage. +3. **Proactively suggest the `techlinHost` and `techlinKey` custom properties** (the + Custom Properties group) — they point the capability at the Collibra Data Lineage + (techlin) server. See the per-source reference for details. + ## Trigger and track — 202 means submitted, not finished `start_technical_lineage` wraps a bare POST that returns `202 Accepted` with no body: diff --git a/pkg/skills/files/collibra/tech-lineage/references/snowflake.md b/pkg/skills/files/collibra/tech-lineage/references/snowflake.md index f1cbe86..d7d6f9b 100644 --- a/pkg/skills/files/collibra/tech-lineage/references/snowflake.md +++ b/pkg/skills/files/collibra/tech-lineage/references/snowflake.md @@ -27,21 +27,31 @@ succeeds. ## Capability parameters — Main Properties +**Ask `snowflakeMode` first** — it decides which of the remaining parameters exist and +which are required, so nothing else should be collected before it: + +- `snowflakeMode` (required, **first question**) — how lineage is derived: + - **`SQL`** — parses the SQL of views and stored procedures. Gives lineage for + defined objects; requires `databaseNames`. + - **`SQL-API`** — reads `ACCESS_HISTORY` + `OBJECT_DEPENDENCIES`, i.e. lineage from + what actually ran, including ad-hoc DML. Supports `extraDatabaseDefinitions` and + `snowflakeDays`. + +Then the rest: + - `id` (required) — **Source ID**: a unique identifier for this lineage source. It keys the source on the lineage server: re-running with the same `id` replaces that source's lineage; other sources can reference it via `dependentSourceIds`. Pick a stable, - descriptive value (e.g. `snowflake-prod`). + descriptive value (e.g. `snowflake-prod`) and **verify it is unique** — it must not + collide with any other lineage source's `id` (check existing lineage capabilities' + `id` parameter via `edge_list_capabilities`), nor with the `collibraSystemName`, nor + with a database or schema name. A collision overwrites or cross-links another + source's lineage. - `connection` (required) — the Generic JDBC connection id (from `get_registered_database`). - `collibraSystemName` (required) — the System asset name used to stitch harvested lineage to catalog assets. Must match the system under which jdbc-ingestion registered the database's assets. -- `snowflakeMode` (required) — how lineage is derived: - - **`SQL`** — parses the SQL of views and stored procedures. Gives lineage for - defined objects; requires `databaseNames`. - - **`SQL-API`** — reads `ACCESS_HISTORY` + `OBJECT_DEPENDENCIES`, i.e. lineage from - what actually ran, including ad-hoc DML. Supports `extraDatabaseDefinitions` and - `snowflakeDays`. - `databaseNames` (list; required in `SQL` mode) — the databases to harvest. Collect the names from the user directly; leave the manifest's `databaseNamesJson` file-upload variant unset (there are no files in this flow). @@ -59,6 +69,20 @@ One editable SQL parameter per query type (`columns`, `views`, `procedures`, and them (e.g. custom filtering); placeholders like `##DBNAMES##`, `##SCHEMANAMES##`, `##DAYS##` are substituted at run time. +## Custom Properties — suggest these proactively + +The Custom Properties group takes free-form key/value pairs the harvester reads at run +time. **Suggest adding these two** when creating the capability: + +- `techlinHost` — host URL of the Collibra Data Lineage (techlin) server the harvested + batches are uploaded to. +- `techlinKey` — the API key for that server. + +The values are tenant-specific (the lineage server assigned to the customer's +environment) — ask the user for them; if they don't have them at hand, the capability +can be created without and updated later (`edge_create_capability` upserts by +`capabilityId`). + ## Advanced Properties - `processingLevel` — `Load` (harvest only, raw metadata inspectable), `Analyze` From f5177660f4d1800eb2e96c3478527272e374de0a Mon Sep 17 00:00:00 2001 From: Yauheni Hlazkou Date: Thu, 9 Jul 2026 14:23:44 +0200 Subject: [PATCH 3/6] feat: capability parameter guidance for tech-lineage skill (mode-first, unique source id, techlin custom properties, mode-dependent queries) --- .../tech-lineage/references/snowflake.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/skills/files/collibra/tech-lineage/references/snowflake.md b/pkg/skills/files/collibra/tech-lineage/references/snowflake.md index d7d6f9b..731248b 100644 --- a/pkg/skills/files/collibra/tech-lineage/references/snowflake.md +++ b/pkg/skills/files/collibra/tech-lineage/references/snowflake.md @@ -61,13 +61,20 @@ Then the rest: - `extraDatabaseDefinitions` (list, `SQL-API` only) — databases to load metadata for without harvesting lineage from them (targets of cross-database references). -## Queries group +## Queries group — mode-dependent, ask after the mode is chosen -One editable SQL parameter per query type (`columns`, `views`, `procedures`, and in -`SQL-API` mode `object_dependencies`, `access_history`, …), pre-filled with the default -`ACCOUNT_USAGE` queries. Leave the defaults unless the user explicitly needs to override -them (e.g. custom filtering); placeholders like `##DBNAMES##`, `##SCHEMANAMES##`, -`##DAYS##` are substituted at run time. +Each query type is an editable SQL parameter pre-filled with the default +`ACCOUNT_USAGE` query. Once `snowflakeMode` is settled, **ask whether the user wants to +configure any of the harvest queries** (most should keep the defaults). If yes, offer +**only the query types of the chosen mode**: + +- **`SQL` mode:** `columns`, `views`, `dynamic_tables`, `semantic_views`, `procedures`. +- **`SQL-API` mode:** `external_stages`, `object_dependencies`, `columns_joined`. + +Never offer the other mode's query types. Placeholders like `##DBNAMES##`, +`##SCHEMANAMES##`, `##DAYS##` are substituted at run time and must be preserved in any +override; the live manifest remains authoritative for the exact query parameter names +in the installed capability version. ## Custom Properties — suggest these proactively From bbf2a0c0c7bd3ed9bed5b6be1cabbf556ae19243 Mon Sep 17 00:00:00 2001 From: Yauheni Hlazkou Date: Thu, 9 Jul 2026 16:04:16 +0200 Subject: [PATCH 4/6] fix: update tool references after configure_database split into register_database --- pkg/skills/files/collibra/tech-lineage/SKILL.md | 2 +- pkg/tools/get_registered_database/tool.go | 6 +++--- pkg/tools/start_technical_lineage/tool.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/skills/files/collibra/tech-lineage/SKILL.md b/pkg/skills/files/collibra/tech-lineage/SKILL.md index 1fe0663..dc01f8a 100644 --- a/pkg/skills/files/collibra/tech-lineage/SKILL.md +++ b/pkg/skills/files/collibra/tech-lineage/SKILL.md @@ -21,7 +21,7 @@ own lineage sync capabilities managed as ETL integrations instead (Databricks, G Technical lineage builds on top of a completed jdbc-ingestion: 1. A **Database asset** must exist for the data source, registered via - `configure_database` (`collibra/jdbc-ingestion`). Its UUID is the key that triggers + `register_database` (`collibra/jdbc-ingestion`). Its UUID is the key that triggers the harvest. 2. **JDBC ingestion must have run** (`start_ingestion`) — harvested lineage stitches to the ingested Schema/Table/Column assets; without them the lineage has nothing to diff --git a/pkg/tools/get_registered_database/tool.go b/pkg/tools/get_registered_database/tool.go index c4bdb92..f473f61 100644 --- a/pkg/tools/get_registered_database/tool.go +++ b/pkg/tools/get_registered_database/tool.go @@ -14,7 +14,7 @@ import ( ) type Input struct { - AssetID string `json:"assetId" jsonschema:"UUID of the registered Database asset (as created by configure_database)."` + AssetID string `json:"assetId" jsonschema:"UUID of the registered Database asset (as created by register_database)."` } type Output struct { @@ -28,7 +28,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "get_registered_database", Title: "Get Registered Database", - Description: "Resolves a registered Database asset to the Edge connection behind it (database → database connection → Edge connection), returning the connection and its edgeSiteId. Use it to reuse the ingestion's connection when setting up dependent capabilities such as Technical Lineage — the capability must be created on the returned edgeSiteId. Fails if the asset is not a database registered via configure_database.", + Description: "Resolves a registered Database asset to the Edge connection behind it (database → database connection → Edge connection), returning the connection and its edgeSiteId. Use it to reuse the ingestion's connection when setting up dependent capabilities such as Technical Lineage — the capability must be created on the returned edgeSiteId. Fails if the asset is not a database registered via register_database.", Handler: handler(collibraClient), Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, @@ -43,7 +43,7 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { database, err := clients.GetDatabase(ctx, collibraClient, input.AssetID) if err != nil { - return Output{Success: false, Error: fmt.Sprintf("failed to get registered database (is this the UUID of a Database asset registered via configure_database?): %s", err.Error())}, nil + return Output{Success: false, Error: fmt.Sprintf("failed to get registered database (is this the UUID of a Database asset registered via register_database?): %s", err.Error())}, nil } if database.DatabaseConnectionID == "" { return Output{Database: database, Success: false, Error: "registered database has no database connection"}, nil diff --git a/pkg/tools/start_technical_lineage/tool.go b/pkg/tools/start_technical_lineage/tool.go index 6e48164..b10deef 100644 --- a/pkg/tools/start_technical_lineage/tool.go +++ b/pkg/tools/start_technical_lineage/tool.go @@ -14,7 +14,7 @@ import ( ) type Input struct { - AssetID string `json:"assetId" jsonschema:"UUID of the Database asset to harvest technical lineage for — the asset registered by jdbc-ingestion (configure_database), NOT the capability id. A Technical Lineage capability referencing the database's connection must already exist on the edge site."` + AssetID string `json:"assetId" jsonschema:"UUID of the Database asset to harvest technical lineage for — the asset registered by jdbc-ingestion (register_database), NOT the capability id. A Technical Lineage capability referencing the database's connection must already exist on the edge site."` } type Output struct { @@ -26,7 +26,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "start_technical_lineage", Title: "Start Technical Lineage Harvest", - Description: "Triggers the technical lineage harvest for a registered Database asset. The database must already be registered (configure_database) and ingested (start_ingestion), and a Technical Lineage capability must exist on the edge site. This is the only correct trigger for technical lineage — never use edge_run_capability or catalog_etl_start_job for it. A success response only means the harvest was submitted (the endpoint returns no job id): locate the spawned DGC job with jobs_find and poll it with get_job_status (NOT edge_get_job_status — the Edge-side harvest is tracked through the DGC job).", + Description: "Triggers the technical lineage harvest for a registered Database asset. The database must already be registered (register_database) and ingested (start_ingestion), and a Technical Lineage capability must exist on the edge site. This is the only correct trigger for technical lineage — never use edge_run_capability or catalog_etl_start_job for it. A success response only means the harvest was submitted (the endpoint returns no job id): locate the spawned DGC job with jobs_find and poll it with get_job_status (NOT edge_get_job_status — the Edge-side harvest is tracked through the DGC job).", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, From 745760c86476df4c67c8b38eefc9289b4ed2f8e9 Mon Sep 17 00:00:00 2001 From: Yauheni Hlazkou Date: Thu, 9 Jul 2026 23:36:17 +0200 Subject: [PATCH 5/6] feat: techlin skill restructure with per-source references and runtime-truth parameter guidance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure collibra/tech-lineage into collibra/techlin with an etl-integration-style per-source catalog (references/index.md), and correct the skill content against the harvester runtime and live Edge behavior: - databaseNames is required in BOTH snowflakeMode values (manifest marks it optional; Edge accepts a capability without it and the harvest then fails asynchronously with "No database names provided") - fix the mode/query mapping: external_stages is SQL-mode, access_history (required, previously undocumented) is SQL-API, dynamic_tables and semantic_views are mode-independent - query defaults are automatic: omit the parameter (Edge materializes required defaults at save) or pass the "use-default" sentinel — never collect or invent SQL - techlinHost/techlinKey must be encoded inside the customParameters list parameter ({name, value, type, secret, encrypted, fromVault}); top-level keys are silently ignored by the harvester - reframe edge_list_capability_types as an inventory: manifests carry existence/names/options, not requiredness or mode gating (no conditional metadata exists in the manifest format); the per-source reference owns the runtime semantics - document Edge save behavior in the skill and tool descriptions: required defaults are materialized, and a second capability of the same type on one connection is rejected with 400 "already used" (treat as exists, never switch connections) --- SKILLS.md | 2 +- .../files/collibra/etl-integration/SKILL.md | 8 +- .../etl-integration/references/index.md | 2 +- pkg/skills/files/collibra/index/SKILL.md | 6 +- .../files/collibra/jdbc-ingestion/SKILL.md | 4 +- pkg/skills/files/collibra/lineage/SKILL.md | 6 +- .../files/collibra/tech-lineage/SKILL.md | 147 ---------- .../tech-lineage/references/snowflake.md | 120 --------- pkg/skills/files/collibra/techlin/SKILL.md | 251 ++++++++++++++++++ .../collibra/techlin/references/index.md | 29 ++ .../collibra/techlin/references/snowflake.md | 178 +++++++++++++ pkg/tools/edge_create_capability/tool.go | 4 +- pkg/tools/edge_list_capability_types/tool.go | 2 +- 13 files changed, 478 insertions(+), 281 deletions(-) delete mode 100644 pkg/skills/files/collibra/tech-lineage/SKILL.md delete mode 100644 pkg/skills/files/collibra/tech-lineage/references/snowflake.md create mode 100644 pkg/skills/files/collibra/techlin/SKILL.md create mode 100644 pkg/skills/files/collibra/techlin/references/index.md create mode 100644 pkg/skills/files/collibra/techlin/references/snowflake.md diff --git a/SKILLS.md b/SKILLS.md index 95722a3..228661a 100644 --- a/SKILLS.md +++ b/SKILLS.md @@ -19,7 +19,7 @@ those tools. Skill content lives in [`pkg/skills/files/collibra/`](pkg/skills/fi | `collibra/asset-edit` | `edit_asset` operation types | | `collibra/jdbc-ingestion` | End-to-end jdbc-ingestion setup: Edge connection/capability, database registration, running and polling a sync | | `collibra/etl-integration` | End-to-end ETL sync integrations (Dataplex, Databricks UC, Purview, Sigma, …): connection, capability, generic config (fetched live), schedule/run; per-integration references | -| `collibra/tech-lineage` | Technical lineage harvesting setup: prerequisites (Database asset + jdbc-ingestion), Technical Lineage capability, `start_technical_lineage` trigger; per-source references (Snowflake) | +| `collibra/techlin` | Technical lineage (techlin) harvesting setup: prerequisites (Database asset + jdbc-ingestion), Technical Lineage capability, `start_technical_lineage` trigger; per-source references (Snowflake) | Each skill is one `SKILL.md` per directory, with frontmatter (`description`, `related`) and an optional `references/` directory for bundled reference documents. diff --git a/pkg/skills/files/collibra/etl-integration/SKILL.md b/pkg/skills/files/collibra/etl-integration/SKILL.md index e32cf68..2800b2f 100644 --- a/pkg/skills/files/collibra/etl-integration/SKILL.md +++ b/pkg/skills/files/collibra/etl-integration/SKILL.md @@ -1,6 +1,6 @@ --- description: Set up, configure, schedule, and run Edge ETL integrations (metadata sync capabilities like Dataplex, Databricks Unity Catalog, Purview, Sigma) end to end. -related: collibra/jdbc-ingestion, collibra/discovery, collibra/tech-lineage +related: collibra/jdbc-ingestion, collibra/discovery, collibra/techlin --- # ETL integrations — connect, configure, schedule, run @@ -28,6 +28,12 @@ which ingestion type it should be** — don't assume ETL just because this skill Only proceed with this ETL skill once the user has chosen it (or it's the only supported path). +**Technical Lineage capabilities are not ETL integrations.** If the capability to create +or run is a technical-lineage one (display name "Technical Lineage for ", e.g. +Snowflake), stop here — that whole flow, including creating the capability and its +mandatory parameter conversation, is owned by `collibra/techlin`. Load that skill +instead; do not create the capability from this one. + ## The three configuration layers (merged at run time) When a run starts, DGC's catalog module merges three parameter sets into one and sends diff --git a/pkg/skills/files/collibra/etl-integration/references/index.md b/pkg/skills/files/collibra/etl-integration/references/index.md index 3c3aee8..2fc9651 100644 --- a/pkg/skills/files/collibra/etl-integration/references/index.md +++ b/pkg/skills/files/collibra/etl-integration/references/index.md @@ -36,7 +36,7 @@ through the `catalog_etl_*` tools — see "Not controllable via chip (yet)" in ` Notes: - **JDBC technical lineage capabilities (e.g. "Technical Lineage for Snowflake") are not `catalog_etl_*` integrations** — they are triggered per Database asset via - `start_technical_lineage`. See `collibra/tech-lineage`. + `start_technical_lineage`. See `collibra/techlin`. - ThoughtSpot's manifest uses a generated typeId; confirm the live value with `edge_list_capability_types` if `edge_create_capability` rejects the id above. - SAP capabilities (AI Core, Datasphere, Data Products, S/4HANA) are out of scope for this diff --git a/pkg/skills/files/collibra/index/SKILL.md b/pkg/skills/files/collibra/index/SKILL.md index a4b9285..9b63a91 100644 --- a/pkg/skills/files/collibra/index/SKILL.md +++ b/pkg/skills/files/collibra/index/SKILL.md @@ -1,6 +1,6 @@ --- description: Navigator for chip's Collibra skills. Start here when unsure which skill applies. -related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/jdbc-ingestion, collibra/etl-integration, collibra/tech-lineage +related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/jdbc-ingestion, collibra/etl-integration, collibra/techlin --- # Collibra skills — navigator @@ -21,12 +21,12 @@ must be bridged to another, and which permissions are required. | Generate governed YAML context (semantic blueprints, metric definitions) for an asset | `collibra/context` | | Set up an Edge JDBC connection/capability and run jdbc-ingestion | `collibra/jdbc-ingestion` | | Set up any metadata-sync ETL integration (Dataplex, Databricks UC, Purview, Sigma, …) — connection, capability, generic config, schedule/run | `collibra/etl-integration` | -| **Create, set up, enable, or run technical lineage** for a registered database (Snowflake, …) | `collibra/tech-lineage` | +| **Create, set up, enable, or run technical lineage** for a registered database (Snowflake, …) | `collibra/techlin` | **"Technical lineage" is ambiguous — route by verb, and don't ask the user which skill to use.** *Querying* what already exists (trace, upstream, downstream, impact, "where does this come from") → `collibra/lineage`. *Producing* it (create, set up, configure, -enable, run, harvest, ingest lineage for a source) → `collibra/tech-lineage`. If the +enable, run, harvest, ingest lineage for a source) → `collibra/techlin`. If the request genuinely doesn't say (e.g. just "technical lineage for Snowflake"), ask whether they want to explore existing lineage or set up harvesting — never present the skill names themselves as the choice. diff --git a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md index 40873a2..0048307 100644 --- a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md +++ b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md @@ -1,6 +1,6 @@ --- description: Set up an Edge JDBC connection and capability, register a database, and run jdbc-ingestion end to end. -related: collibra/discovery, collibra/tech-lineage +related: collibra/discovery, collibra/techlin --- # JDBC ingestion — connect, register, and run @@ -76,7 +76,7 @@ which ingestion type it should be** — don't assume JDBC just because this skil ingestion finished — always poll to confirm completion. Once ingestion has completed, **technical lineage** can be layered on top of the -registered database — see `collibra/tech-lineage`. +registered database — see `collibra/techlin`. ## Two job-id spaces — do not cross them diff --git a/pkg/skills/files/collibra/lineage/SKILL.md b/pkg/skills/files/collibra/lineage/SKILL.md index 66005b4..c1db9e4 100644 --- a/pkg/skills/files/collibra/lineage/SKILL.md +++ b/pkg/skills/files/collibra/lineage/SKILL.md @@ -1,13 +1,13 @@ --- -description: Query the existing technical lineage graph — trace upstream sources and downstream consumers. Covers the DGC UUID → lineage entity ID bridge. Read-only — for setting up, creating, or running technical lineage harvesting, load collibra/tech-lineage instead. -related: collibra/discovery, collibra/index, collibra/tech-lineage +description: Query the existing technical lineage graph — trace upstream sources and downstream consumers. Covers the DGC UUID → lineage entity ID bridge. Read-only — for setting up, creating, or running technical lineage harvesting, load collibra/techlin instead. +related: collibra/discovery, collibra/index, collibra/techlin --- # Technical lineage — querying the graph This skill only covers **querying** existing lineage. If the user wants to *set up*, *create*, *configure*, *enable*, or *run* technical lineage (harvest lineage from a data -source like Snowflake), **this is the wrong skill — load `collibra/tech-lineage`** +source like Snowflake), **this is the wrong skill — load `collibra/techlin`** before doing anything else. Technical lineage maps the physical data flow — including unregistered assets, temporary diff --git a/pkg/skills/files/collibra/tech-lineage/SKILL.md b/pkg/skills/files/collibra/tech-lineage/SKILL.md deleted file mode 100644 index dc01f8a..0000000 --- a/pkg/skills/files/collibra/tech-lineage/SKILL.md +++ /dev/null @@ -1,147 +0,0 @@ ---- -description: Set up, create, configure, and run technical lineage harvesting for a registered database (Snowflake, …) — resolve the connection, create the Technical Lineage capability, trigger and track the harvest. Use for any "create/set up/enable/run technical lineage" request; for querying existing lineage, load collibra/lineage instead. -related: collibra/jdbc-ingestion, collibra/etl-integration, collibra/lineage, collibra/discovery ---- - -# Technical lineage — prerequisites, capability, harvest - -Technical lineage extracts data-flow lineage (views, procedures, query history) from a -data source and loads it into Collibra's lineage graph. On Edge it is a **capability** -(display name "Technical Lineage for ") that reads the source through the same -**Generic JDBC connection** used for jdbc-ingestion, and is triggered through a dedicated -catalog endpoint keyed by the **Database asset** — not by the capability. - -This skill is about *producing* technical lineage. For *querying* the lineage graph -(upstream/downstream, transformations), see `collibra/lineage`. Some sources have their -own lineage sync capabilities managed as ETL integrations instead (Databricks, GCP/Dataplex) -— those go through `collibra/etl-integration`, not this skill. - -## Prerequisites — check these first, before any setup - -Technical lineage builds on top of a completed jdbc-ingestion: - -1. A **Database asset** must exist for the data source, registered via - `register_database` (`collibra/jdbc-ingestion`). Its UUID is the key that triggers - the harvest. -2. **JDBC ingestion must have run** (`start_ingestion`) — harvested lineage stitches to - the ingested Schema/Table/Column assets; without them the lineage has nothing to - attach to. - -Verify before proceeding: resolve the Database asset with `search_asset_keyword` -(filter by asset type `"Database"`; `collibra/discovery` has the name→UUID rules). If no -Database asset exists, or ingestion never ran: **stop and tell the user** that a -registered database and a completed JDBC ingestion are prerequisites for technical -lineage, and offer to set them up first via `collibra/jdbc-ingestion`. Do not try to -create the Database asset any other way. - -## The full sequence - -1. **Resolve the Database asset** (see Prerequisites) → keep its UUID; it is both the - input of step 2 and the `assetId` for step 5. -2. **`get_registered_database`** with that UUID → the **Edge connection** backing the - ingestion and its **`edgeSiteId`**. Both are derived, not chosen: the connection id - feeds the capability's `connection` parameter, and the capability **must be created - on that same edge site**. A failure here means the asset is not a registered - database — treat it as a missing prerequisite (see above). Only if this tool is - unavailable, fall back to `edge_list_sites` + `edge_find_connections` and confirm - the picks with the user. -3. **`edge_list_capability_types`** (the `edgeSiteId` from step 2; pass a `query`, e.g. - `"snowflake"`) → find the **Technical Lineage** capability type and read its - parameter manifest. **Several capability types can match the data source name — only - the technical-lineage one is correct.** See "Exactly one right capability type". -4. **`edge_create_capability`** → `typeId` from step 3, `edgeSiteId` and - `parameters.connection` from step 2, remaining `parameters` from the manifest — ask - the user for the values and confirm the full set first. See "Capability parameters". - Skip if a Technical Lineage capability for this source already exists - (`edge_find_capabilities`). -5. **`start_technical_lineage`** with the Database asset UUID from step 1 — **not** the - capability id. Success means *submitted*, nothing more. -6. **Track:** the trigger returns no job id. Find the spawned DGC job with **`jobs_find`** - (jobs started at/after the trigger), then poll it with **`get_job_status`** until it - reaches a terminal state. Never poll with `edge_get_job_status` — the Edge-side - harvest is tracked through the DGC job. - -## Exactly one right capability type - -An Edge site lists many capability types, and more than one can contain the data-source -name (e.g. jdbc-ingestion profiles, sync capabilities, *and* technical lineage for -Snowflake). The technical-lineage capability is recognizable by its identity, not by the -name substring alone — for Snowflake it is the type whose manifest displays -**"Technical Lineage for Snowflake"** (see `references/snowflake.md` for the expected -`typeId`). If the expected type is not present on the site, list what -`edge_list_capability_types` returned and ask the user — never fall back to another -`*snowflake*` type. - -In particular, **capability types for SQL files in cloud storage** (lineage harvested -from `.sql` files in an S3/ADLS/GCS bucket) can also carry the dialect name and their -manifests ask for file/storage locations. Those belong to the separate "lineage for -cloud storage" flow — see "Not supported" below. This skill's flow harvests **directly -from the database** over the JDBC connection; there are no files to locate, so never -ask the user where files are stored. - -## Not supported: lineage for cloud storage - -Technical lineage from **SQL files in cloud storage** (S3, ADLS, GCS buckets) is a -different flow with different capability types and parameters, and is **not supported -via chip yet**. If the user's lineage source is SQL files in a bucket rather than a -registered database, tell them it must be configured in the Collibra/Edge UI — do not -attempt it with this skill's tools or pick a cloud-storage capability type. - -## Capability parameters - -Read the parameter names, requiredness, defaults, and options from the **live manifest** -(`edge_list_capability_types` with a `query`) — never from memory; manifests evolve with -the capability version. The per-source reference (`references/snowflake.md`) explains -what the fields *mean* and how to choose values; the manifest stays authoritative for -exact names and defaults. - -To fill them in: ask the user for the required values, propose defaults for the -optional ones, and **confirm the complete parameter set with the user before calling -`edge_create_capability`**. Never ask for database credentials — they live in the Edge -connection, not in capability parameters. - -Ordering and rules for the conversation: - -1. **If the source has an ingestion-mode parameter (Snowflake: `snowflakeMode`), settle - it first** — the mode decides which of the remaining parameters exist and which are - required, so asking anything else before it wastes the user's time. Re-check the - manifest against the chosen mode before continuing. -2. **Source ID must be unique.** Propose a value, but never one that collides with an - existing lineage source id (check the `id` parameter of existing lineage - capabilities via `edge_list_capabilities`/`edge_find_capabilities`), with the - `collibraSystemName`, or with a database/schema name. A colliding Source ID silently - overwrites or cross-links another source's lineage. -3. **Proactively suggest the `techlinHost` and `techlinKey` custom properties** (the - Custom Properties group) — they point the capability at the Collibra Data Lineage - (techlin) server. See the per-source reference for details. - -## Trigger and track — 202 means submitted, not finished - -`start_technical_lineage` wraps a bare POST that returns `202 Accepted` with no body: - -- A success response **only** means DGC accepted the harvest request. -- No job id is returned. Locate the spawned DGC job with `jobs_find` right after the - call (filter to jobs started at/after the trigger) and poll it with `get_job_status` - to a terminal state before reporting the harvest as done. If the job ends in failure, - surface its message/status log to the user. - -## Hard rules - -1. **Verify the prerequisites first** (Database asset + completed ingestion). When - missing, stop and route to `collibra/jdbc-ingestion` — don't improvise. -2. **Read capability parameters from `edge_list_capability_types`,** not memory; use - `references/.md` for meaning, the manifest for names/defaults. -3. **Ask the user for parameter values and confirm the full set** before - `edge_create_capability`. Never ask for credentials. The `connection` value and the - `edgeSiteId` are not user choices — take both from `get_registered_database`; the - capability lives on the same edge site as the connection. -4. **Only `start_technical_lineage` triggers a harvest** — never `edge_run_capability` - and never `catalog_etl_start_job`. Pass the **Database asset** UUID, not the - capability id. -5. **Always locate and poll the DGC job after triggering** (`jobs_find` → - `get_job_status`). A 202/success alone is not a completed harvest. -6. **When several capability types match the source name, never pick one silently** — - match the technical-lineage identity or ask. -7. **Databases only.** This flow harvests from the database over the JDBC connection — - never ask where files are stored. Lineage for SQL files in cloud storage is not - supported via chip; route those users to the Collibra/Edge UI. \ No newline at end of file diff --git a/pkg/skills/files/collibra/tech-lineage/references/snowflake.md b/pkg/skills/files/collibra/tech-lineage/references/snowflake.md deleted file mode 100644 index 731248b..0000000 --- a/pkg/skills/files/collibra/tech-lineage/references/snowflake.md +++ /dev/null @@ -1,120 +0,0 @@ -# Technical Lineage for Snowflake (`edgeharvester-snowflake`) - -Harvests lineage from Snowflake — view/procedure SQL or `ACCESS_HISTORY` — and loads it -into the Collibra lineage graph, stitched to the assets ingested by jdbc-ingestion. - -- **Capability typeId** (for `edge_create_capability`): **`edgeharvester-snowflake`** — - the manifest displays **"Technical Lineage for Snowflake"**. Confirm the live id with - `edge_list_capability_types(query: "snowflake")`; do not confuse it with other - Snowflake-named capability types on the site — especially not with the SQL-files / - cloud-storage lineage variants, whose manifests ask for file or bucket locations - (that flow is not supported via chip). -- **Connection:** the Generic JDBC connection (`typeId: "Generic"`) created for - jdbc-ingestion — resolve it (and the edge site for the capability) from the Database - asset with `get_registered_database`. Credentials, role, and warehouse live in the - connection — the capability never carries them. -- **Trigger:** `start_technical_lineage(assetId)` with the **Database asset** UUID. - -**Docs:** [Technical lineage for Snowflake](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/Snowflake/to_snowflake.htm) - -## Snowflake access requirements - -The connection's role needs access to `SNOWFLAKE.ACCOUNT_USAGE` views (`TABLES`, -`COLUMNS`, `VIEWS`, `PROCEDURES`, `OBJECT_DEPENDENCIES`, `ACCESS_HISTORY`, -`QUERY_HISTORY`, …) — typically granted with `IMPORTED PRIVILEGES` on the shared -`SNOWFLAKE` database. Without it the harvest queries fail even though `test_connection` -succeeds. - -## Capability parameters — Main Properties - -**Ask `snowflakeMode` first** — it decides which of the remaining parameters exist and -which are required, so nothing else should be collected before it: - -- `snowflakeMode` (required, **first question**) — how lineage is derived: - - **`SQL`** — parses the SQL of views and stored procedures. Gives lineage for - defined objects; requires `databaseNames`. - - **`SQL-API`** — reads `ACCESS_HISTORY` + `OBJECT_DEPENDENCIES`, i.e. lineage from - what actually ran, including ad-hoc DML. Supports `extraDatabaseDefinitions` and - `snowflakeDays`. - -Then the rest: - -- `id` (required) — **Source ID**: a unique identifier for this lineage source. It keys - the source on the lineage server: re-running with the same `id` replaces that source's - lineage; other sources can reference it via `dependentSourceIds`. Pick a stable, - descriptive value (e.g. `snowflake-prod`) and **verify it is unique** — it must not - collide with any other lineage source's `id` (check existing lineage capabilities' - `id` parameter via `edge_list_capabilities`), nor with the `collibraSystemName`, nor - with a database or schema name. A collision overwrites or cross-links another - source's lineage. -- `connection` (required) — the Generic JDBC connection id (from - `get_registered_database`). -- `collibraSystemName` (required) — the System asset name used to stitch harvested - lineage to catalog assets. Must match the system under which jdbc-ingestion registered - the database's assets. -- `databaseNames` (list; required in `SQL` mode) — the databases to harvest. Collect the - names from the user directly; leave the manifest's `databaseNamesJson` file-upload - variant unset (there are no files in this flow). -- `schemaNames` (list, optional) — restrict harvesting to these schemas. -- `snowflakeDays` (default `365`) — how many days of `ACCESS_HISTORY` to read - (`SQL-API` mode). Lower it for faster harvests on busy accounts. -- `extraDatabaseDefinitions` (list, `SQL-API` only) — databases to load metadata for - without harvesting lineage from them (targets of cross-database references). - -## Queries group — mode-dependent, ask after the mode is chosen - -Each query type is an editable SQL parameter pre-filled with the default -`ACCOUNT_USAGE` query. Once `snowflakeMode` is settled, **ask whether the user wants to -configure any of the harvest queries** (most should keep the defaults). If yes, offer -**only the query types of the chosen mode**: - -- **`SQL` mode:** `columns`, `views`, `dynamic_tables`, `semantic_views`, `procedures`. -- **`SQL-API` mode:** `external_stages`, `object_dependencies`, `columns_joined`. - -Never offer the other mode's query types. Placeholders like `##DBNAMES##`, -`##SCHEMANAMES##`, `##DAYS##` are substituted at run time and must be preserved in any -override; the live manifest remains authoritative for the exact query parameter names -in the installed capability version. - -## Custom Properties — suggest these proactively - -The Custom Properties group takes free-form key/value pairs the harvester reads at run -time. **Suggest adding these two** when creating the capability: - -- `techlinHost` — host URL of the Collibra Data Lineage (techlin) server the harvested - batches are uploaded to. -- `techlinKey` — the API key for that server. - -The values are tenant-specific (the lineage server assigned to the customer's -environment) — ask the user for them; if they don't have them at hand, the capability -can be created without and updated later (`edge_create_capability` upserts by -`capabilityId`). - -## Advanced Properties - -- `processingLevel` — `Load` (harvest only, raw metadata inspectable), `Analyze` - (harvest + analyze), `Sync` (default: also synchronize lineage into the catalog). -- `dependentSourceIds` — Source IDs of other lineage sources this one references, so - cross-source lineage resolves. -- `databaseSystemMapping` — map database names to system names when one capability - covers databases that belong to different systems. -- `deleteRawMetadataAfterProcessing` — remove harvested raw metadata after processing. -- `active` — set to `false` and re-run to **remove** this source's lineage from the - lineage server. -- `techlinAdminConnection` (beta) — optional connection to the lineage (TechLin) server - admin API. Leave unset unless the user asks for it. - -## Gotchas - -- **Stitching needs ingested assets.** Lineage attaches to the Schema/Table/Column - assets created by jdbc-ingestion under `collibraSystemName`; if ingestion never ran - (or ran under a different system name), the harvest succeeds but lineage doesn't link - to catalog assets. -- **The trigger is asset-keyed.** `start_technical_lineage` takes the Database asset - UUID; DGC finds the matching Technical Lineage capability itself. Passing the - capability id fails. -- **`SQL` vs `SQL-API` changes the parameter set** (which query params exist, whether - `databaseNames` is required) — re-read the manifest after the user picks the mode. -- The live manifest from `edge_list_capability_types` is authoritative for exact - parameter names, requiredness, and defaults on the customer's capability version; - this reference explains meaning and choice. \ No newline at end of file diff --git a/pkg/skills/files/collibra/techlin/SKILL.md b/pkg/skills/files/collibra/techlin/SKILL.md new file mode 100644 index 0000000..1413855 --- /dev/null +++ b/pkg/skills/files/collibra/techlin/SKILL.md @@ -0,0 +1,251 @@ +--- +description: Set up, create, configure, and run technical lineage (techlin) harvesting for a registered database (Snowflake, …) — resolve the connection, create and configure the Technical Lineage capability, trigger and track the harvest. Use for any "create/set up/enable/run technical lineage" request; the whole flow, including creating the capability when it is missing, lives here. For querying existing lineage, load collibra/lineage instead. +related: collibra/jdbc-ingestion, collibra/etl-integration, collibra/lineage, collibra/discovery +--- + +# Technical lineage (techlin) — prerequisites, capability, harvest + +Technical lineage — **techlin** for short; the lineage server and the `techlinHost`/ +`techlinKey` properties carry the same name — extracts data-flow lineage (views, +procedures, query history) from a data source and loads it into Collibra's lineage +graph. On Edge it is a **capability** (display name "Technical Lineage for ") +that reads the source through the same **Generic JDBC connection** used for +jdbc-ingestion, and is triggered through a dedicated catalog endpoint keyed by the +**Database asset** — not by the capability. + +Whatever the user asked for — "set up", "run", "harvest" — the flow is the same and +**starts by checking whether the capability exists, never by triggering**. If it +exists, trigger it. If it doesn't, creating and configuring it is part of **this +skill**: load the per-source reference, run the parameter conversation, and create it +here. Never hand the creation off to another skill, and never call +`start_technical_lineage` before the capability exists. + +This skill is about *producing* technical lineage. For *querying* the lineage graph +(upstream/downstream, transformations), see `collibra/lineage`. + +## The catalog-etl parallel — same lifecycle, different tools + +Configuring a techlin capability follows the same approach as an ETL integration +(`collibra/etl-integration`), only spelled with edge tools. There is no separate +"generic config" layer — **the capability's parameters are the entire configuration**: + +| Step | catalog-etl | techlin | +| --- | --- | --- | +| See which parameters exist | `catalog_etl_get_schema` | `edge_list_capability_types` (manifest — an inventory, see below) | +| Save the configuration | `catalog_etl_save_generic_config` | `edge_create_capability` (upserts by `capabilityId`) | +| Read the config back | `catalog_etl_get_config` | `edge_get_capability` | +| Run | `catalog_etl_start_job` / schedules | `start_technical_lineage(assetId)` — no schedules | +| Track | `get_job_status` | `jobs_find` → `get_job_status` | + +One important difference: `catalog_etl_get_schema` returns a JSON Schema the server +enforces when the config is saved; the Edge manifest is **not** enforced as a +contract. At save time Edge materializes defaults for required manifest parameters +(e.g. the harvest queries, `active`) and rejects a second capability of the same type +on one connection — but it accepts any parameter *values* and accepts missing +runtime-required ones (e.g. Snowflake's `databaseNames`). So the dangerous failure +mode is wrong or missing values, and the parameter conversation (step 6 below) is +never optional — a capability created from guesses harvests the wrong thing, or fails +asynchronously in the harvest job. + +## Not this skill: ETL-owned lineage capabilities + +Some sources have lineage capabilities managed as **ETL integrations** by another team +— Databricks Unity Catalog (`databricks-lineage`) and GCP/Dataplex +(`dataplex-lineage-synchronization`). Those use their own connection types (not the +Generic JDBC connection), have no Database-asset prerequisite, and are configured and +scheduled through `catalog_etl_*` — route them to `collibra/etl-integration`. +Conversely, techlin (`edgeharvester-*`) capabilities are never created or run from the +ETL skill. + +## Prerequisites — check these first, before any setup + +Technical lineage builds on top of a completed jdbc-ingestion: + +1. A **Database asset** must exist for the data source, registered via + `register_database` (`collibra/jdbc-ingestion`). Its UUID is the key that triggers + the harvest. +2. **JDBC ingestion must have run** (`start_ingestion`) — harvested lineage stitches to + the ingested Schema/Table/Column assets; without them the lineage has nothing to + attach to. + +Verify before proceeding: resolve the Database asset with `search_asset_keyword` +(filter by asset type `"Database"`; `collibra/discovery` has the name→UUID rules). If no +Database asset exists, or ingestion never ran: **stop and tell the user** that a +registered database and a completed JDBC ingestion are prerequisites for technical +lineage, and offer to set them up first via `collibra/jdbc-ingestion`. Do not try to +create the Database asset any other way. + +## The full sequence + +1. **Resolve the Database asset** (see Prerequisites) → keep its UUID; it is both the + input of step 2 and the `assetId` for step 7. +2. **`get_registered_database`** with that UUID → the **Edge connection** backing the + ingestion and its **`edgeSiteId`**. Both are derived, not chosen: the connection id + feeds the capability's `connection` parameter, and the capability **must be created + on that same edge site**. A failure here means the asset is not a registered + database — treat it as a missing prerequisite (see above). Only if this tool is + unavailable, fall back to `edge_list_sites` + `edge_find_connections` and confirm + the picks with the user. +3. **`edge_find_capabilities`** on that edge site → does a **Technical Lineage + capability for this source** already exist (one whose `connection` parameter is the + step-2 connection)? If yes, skip to step 7. If no, tell the user the capability has + to be created and configured first, and continue with steps 4–6 — the creation + happens **here, in this skill**; do not load another skill for it and do not + trigger anything yet. +4. **Load the per-source reference** — `references/index.md` lists the supported + sources and their capability typeIds; load the source's file (e.g. + `references/snowflake.md`) **before asking the user anything**. It defines which + questions to ask and in what order. If the source has no reference: when it is one + of the ETL-owned lineage capabilities, route to `collibra/etl-integration`; + otherwise walk the step-5 manifest through with the user parameter by parameter — + never guess. +5. **`edge_list_capability_types`** (the `edgeSiteId` from step 2; pass a `query`, e.g. + `"snowflake"`) → find the **Technical Lineage** capability type and read its + parameter manifest — an **inventory**: authoritative for which parameters exist on + the installed version, their exact names, and their options. Requiredness and mode + behavior come from the per-source reference (step 4), not from the manifest's + `required` flags — those describe the Edge UI form. If the manifest lists a + parameter the reference doesn't mention, surface it to the user; never guess. + **Several capability types can match the data source name — only the + technical-lineage one is correct.** See "Exactly one right capability type". +6. **The parameter conversation — a hard checkpoint.** Collect every parameter value + with the user in the order the per-source reference prescribes (mode-gating + question first when the source has one), then present the complete parameter set + and get an explicit go-ahead. Only after the user confirms: + **`edge_create_capability`** → `typeId` from step 5, `edgeSiteId` and + `parameters.connection` from step 2, remaining `parameters` exactly as confirmed. + See "Capability parameters". +7. **`start_technical_lineage`** with the Database asset UUID from step 1 — **not** the + capability id. Only reached once step 3 (or step 6) confirmed the capability exists. + Success means *submitted*, nothing more. +8. **Track:** the trigger returns no job id. Find the spawned DGC job with **`jobs_find`** + (jobs started at/after the trigger), then poll it with **`get_job_status`** until it + reaches a terminal state. Never poll with `edge_get_job_status` — the Edge-side + harvest is tracked through the DGC job. + +## Exactly one right capability type + +An Edge site lists many capability types, and more than one can contain the data-source +name (e.g. jdbc-ingestion profiles, sync capabilities, *and* technical lineage for +Snowflake). The technical-lineage capability is recognizable by its identity, not by the +name substring alone — for Snowflake it is the type whose manifest displays +**"Technical Lineage for Snowflake"** (`references/index.md` lists the supported +sources and their expected typeIds). If the expected type is not present on the site, +list what `edge_list_capability_types` returned and ask the user — never fall back to +another `*snowflake*` type. + +In particular, **capability types for SQL files in cloud storage** (lineage harvested +from `.sql` files in an S3/ADLS/GCS bucket) can also carry the dialect name and their +manifests ask for file/storage locations. Those belong to the separate "lineage for +cloud storage" flow — see "Not supported" below. This skill's flow harvests **directly +from the database** over the JDBC connection; there are no files to locate, so never +ask the user where files are stored. + +## Not supported: lineage for cloud storage + +Technical lineage from **SQL files in cloud storage** (S3, ADLS, GCS buckets) is a +different flow with different capability types and parameters, and is **not supported +via chip yet**. If the user's lineage source is SQL files in a bucket rather than a +registered database, tell them it must be configured in the Collibra/Edge UI — do not +attempt it with this skill's tools or pick a cloud-storage capability type. + +## Capability parameters + +Read the parameter names and options from the **live manifest** +(`edge_list_capability_types` with a `query`) — never from memory; manifests evolve with +the capability version. The per-source reference (via `references/index.md`) is +authoritative for what the fields *mean*, which are required at run time, how the +ingestion mode gates them, and which questions to ask in which order; the manifest's +own `required`/`default` flags describe the Edge UI form, not what the harvest needs. + +**This conversation is the checkpoint between "the capability is missing" and +`edge_create_capability` — it always happens, in full, even when the user has already +said "yes, create it".** That yes authorizes the creation, not skipping the questions. +Never ask for database credentials — they live in the Edge connection, not in +capability parameters. + +Ordering and rules for the conversation: + +1. **Mode first.** If the source has an ingestion-mode parameter (Snowflake: + `snowflakeMode`), settle it before anything else — the mode decides which of the + remaining questions apply, so asking anything else before it wastes the user's + time. The per-source reference documents the gating; the manifest does not encode + it. +2. **Collect every runtime-required parameter the reference lists**, regardless of + the manifest's `required` flags. For Snowflake, `databaseNames` is required in + **both** modes even though the manifest marks it optional — Edge accepts a + capability without it, and the harvest then fails asynchronously with "No database + names provided". +3. **Source ID must be unique.** Propose a value, but never one that collides with an + existing lineage source id (check the `id` parameter of existing lineage + capabilities via `edge_list_capabilities`/`edge_find_capabilities`), with the + `collibraSystemName`, or with a database/schema name. A colliding Source ID silently + overwrites or cross-links another source's lineage. +4. **Proactively offer the `techlinHost` and `techlinKey` custom properties** — they + point the capability at the Collibra Data Lineage (techlin) server and are passed + **inside the `customParameters` list parameter** (the per-source reference shows + the exact encoding), never as top-level parameters. They are optional: ask for + them explicitly; if the user doesn't have the values at hand, deferring them is + the **user's** decision, never a silent default — they can be added later (see + "Updating an existing capability"). +5. **Confirm the complete set.** Present every parameter — chosen values, applied + defaults, custom properties — in one table and get an explicit confirmation before + calling `edge_create_capability`. + +## Updating an existing capability + +Read the current configuration back with `edge_get_capability`; change it by calling +`edge_create_capability` with the same `capabilityId` (it upserts). Start from the +read-back `parameters`, apply only the agreed changes, and send the full set — that +preserves the server-materialized values (harvest queries, `active`) without +re-collecting them. Parameter changes go through the same conversation-and-confirm +rule as creation — show what will change and get a go-ahead first. + +## Trigger and track — 202 means submitted, not finished + +`start_technical_lineage` wraps a bare POST that returns `202 Accepted` with no body: + +- A success response **only** means DGC accepted the harvest request. +- No job id is returned. Locate the spawned DGC job with `jobs_find` right after the + call (filter to jobs started at/after the trigger) and poll it with `get_job_status` + to a terminal state before reporting the harvest as done. If the job ends in failure, + surface its message/status log to the user. + +## Hard rules + +1. **Verify the prerequisites first** (Database asset + completed ingestion). When + missing, stop and route to `collibra/jdbc-ingestion` — don't improvise. +2. **Capability before trigger.** Never call `start_technical_lineage` until + `edge_find_capabilities` has confirmed a Technical Lineage capability exists for the + source's connection. Triggering without one sets nothing up — it fails or spawns a + failed job. When it's missing, say so and switch to the creation conversation. +3. **Creating the capability is this skill's job.** A techlin capability is not an ETL + integration — never load `collibra/etl-integration` (or any other skill) to create + it, and never use `catalog_etl_*` tools on it. The ETL-owned lineage capabilities + (`databricks-lineage`, `dataplex-lineage-synchronization`) are the one exception — + route those to `collibra/etl-integration`. +4. **Load the per-source reference before the parameter conversation** + (`references/index.md` → `references/.md`) and **read the live manifest** + (`edge_list_capability_types`) — the reference for meaning, question order, and + requiredness; the manifest for exact names and options; never memory. Manifest + parameters the reference doesn't mention are surfaced to the user, not guessed. +5. **Ask the user for parameter values and confirm the full set** before + `edge_create_capability` — even when the user already approved creating the + capability; approval covers the creation, not skipping the questions. Never ask for + credentials. The `connection` value and the `edgeSiteId` are not user choices — + take both from `get_registered_database`; the capability lives on the same edge + site as the connection. +6. **Only `start_technical_lineage` triggers a harvest** — never `edge_run_capability` + and never `catalog_etl_start_job`. Pass the **Database asset** UUID, not the + capability id. +7. **Always locate and poll the DGC job after triggering** (`jobs_find` → + `get_job_status`). A 202/success alone is not a completed harvest. +8. **When several capability types match the source name, never pick one silently** — + match the technical-lineage identity or ask. +9. **Databases only.** This flow harvests from the database over the JDBC connection — + never ask where files are stored. Lineage for SQL files in cloud storage is not + supported via chip; route those users to the Collibra/Edge UI. +10. **One capability per (type, connection).** When `edge_create_capability` fails + with `400 — "Connection(s) […] already used in the capability of the same type"`, + the capability already exists — go back to `edge_find_capabilities` and the + update or trigger path. Never retry with a different connection. \ No newline at end of file diff --git a/pkg/skills/files/collibra/techlin/references/index.md b/pkg/skills/files/collibra/techlin/references/index.md new file mode 100644 index 0000000..a31aaf0 --- /dev/null +++ b/pkg/skills/files/collibra/techlin/references/index.md @@ -0,0 +1,29 @@ +# Technical lineage (techlin) source catalog + +Every source below is harvested through the same lifecycle (see the parent +`SKILL.md`): `get_registered_database` → `edge_find_capabilities` → +`edge_list_capability_types` → the parameter conversation → `edge_create_capability` +→ `start_technical_lineage` → `jobs_find`/`get_job_status`. + +The **capability typeId** is what you pass to `edge_create_capability` (`typeId`). +Always confirm the live id and the exact parameter names/options via +`edge_list_capability_types`; the per-source reference is authoritative for +requiredness, mode gating, and question order — the manifest's `required` flags +describe the Edge UI form, not what the harvest needs. + +| Source | capability typeId | Reference | Notes | +| --- | --- | --- | --- | +| Snowflake | `edgeharvester-snowflake` | `snowflake.md` | two modes (`snowflakeMode`: `SQL` / `SQL-API`) — the mode gates the questions; the gating is documented in `snowflake.md`, not encoded in the manifest | + +More sources (Oracle, SAP HANA, …) will be added here as their capabilities ship — +one reference file per source, same lifecycle, no `SKILL.md` change needed. + +**Not in this catalog:** + +- **Databricks Unity Catalog lineage (`databricks-lineage`) and GCP/Dataplex lineage + (`dataplex-lineage-synchronization`)** are ETL integrations owned by another team: + own connection types, no Database-asset prerequisite, configured and scheduled via + `catalog_etl_*`. Route them to `collibra/etl-integration`. +- **SQL files in cloud storage** (S3/ADLS/GCS buckets) — a different flow with + different capability types and parameters, not supported via chip; route users to + the Collibra/Edge UI. \ No newline at end of file diff --git a/pkg/skills/files/collibra/techlin/references/snowflake.md b/pkg/skills/files/collibra/techlin/references/snowflake.md new file mode 100644 index 0000000..e67804d --- /dev/null +++ b/pkg/skills/files/collibra/techlin/references/snowflake.md @@ -0,0 +1,178 @@ +# Technical Lineage for Snowflake (`edgeharvester-snowflake`) + +Harvests lineage from Snowflake — view/procedure SQL or `ACCESS_HISTORY` — and loads it +into the Collibra lineage graph, stitched to the assets ingested by jdbc-ingestion. + +- **Capability typeId** (for `edge_create_capability`): **`edgeharvester-snowflake`** — + the manifest displays **"Technical Lineage for Snowflake"**. Confirm the live id with + `edge_list_capability_types(query: "snowflake")` — the query also matches other + Snowflake-named types on the site (e.g. the ETL-owned `snowflake-synchronization`, + Collibra Protect); only the technical-lineage one is correct. The SQL-files / + cloud-storage lineage variants (manifests asking for file or bucket locations) are + not supported via chip. +- **Connection:** the Generic JDBC connection (`typeId: "Generic"`) created for + jdbc-ingestion — resolve it (and the edge site for the capability) from the Database + asset with `get_registered_database`. Credentials, role, and warehouse live in the + connection — the capability never carries them. +- **Trigger:** `start_technical_lineage(assetId)` with the **Database asset** UUID. + +**Docs:** [Technical lineage for Snowflake](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/Snowflake/to_snowflake.htm) + +## Snowflake access requirements + +The connection's role needs access to `SNOWFLAKE.ACCOUNT_USAGE` views (`TABLES`, +`COLUMNS`, `VIEWS`, `PROCEDURES`, `OBJECT_DEPENDENCIES`, `ACCESS_HISTORY`, +`QUERY_HISTORY`, …) — typically granted with `IMPORTED PRIVILEGES` on the shared +`SNOWFLAKE` database. Without it the harvest queries fail even though `test_connection` +succeeds. + +## The conversation — ask in this order + +This is the question script for the parameter checkpoint (step 6 of the parent +`SKILL.md` sequence). Every question is asked; nothing is filled in silently: + +1. **`snowflakeMode`** — "Should lineage come from parsing view/procedure SQL + (`SQL`), or from what actually ran, via `ACCESS_HISTORY` (`SQL-API`)?" The mode + decides which of the remaining questions apply — the gating is documented here, + in this reference; the manifest does not encode it. +2. **Scope** — **`databaseNames` (required in BOTH modes — see below)** and + `schemaNames` (optional). In `SQL-API` mode additionally: `snowflakeDays` + (default 365) and `extraDatabaseDefinitions`. +3. **`id` (Source ID)** — propose a value, verify uniqueness (see below). +4. **`collibraSystemName`** — must match the system jdbc-ingestion registered the + database's assets under. +5. **Queries** — ask only whether the user wants to *override* any harvest query + (almost nobody does; defaults apply automatically — see "Queries"). +6. **Custom Properties** — offer `techlinHost` + `techlinKey` (optional; note the + `customParameters` encoding below). +7. **Advanced Properties** — only when the user asks for them. +8. **Present the complete parameter set in one table and get an explicit "create + it"** before calling `edge_create_capability`. + +## Capability parameters — Main Properties + +**Ask `snowflakeMode` first** — it decides which of the remaining questions apply. +Requiredness below is the harvest-runtime truth and comes from this reference; the +manifest's `required` flags describe the Edge UI form and disagree with it in both +directions (see Gotchas). The live manifest stays authoritative for exact parameter +names and options on the installed capability version. + +- `snowflakeMode` (required, **first question**) — how lineage is derived: + - **`SQL`** — parses the SQL of views and stored procedures. Gives lineage for + defined objects. + - **`SQL-API`** — reads `ACCESS_HISTORY` + `OBJECT_DEPENDENCIES`, i.e. lineage from + what actually ran, including ad-hoc DML. Supports `snowflakeDays` and + `extraDatabaseDefinitions`. + +Then the rest: + +- `id` (required) — **Source ID**: a unique identifier for this lineage source. It keys + the source on the lineage server: re-running with the same `id` replaces that source's + lineage; other sources can reference it via `dependentSourceIds`. Pick a stable, + descriptive value (e.g. `snowflake-prod`) and **verify it is unique** — it must not + collide with any other lineage source's `id` (check existing lineage capabilities' + `id` parameter via `edge_list_capabilities`), nor with the `collibraSystemName`, nor + with a database or schema name. A collision overwrites or cross-links another + source's lineage. +- `connection` (required) — the Generic JDBC connection id (from + `get_registered_database`). +- `collibraSystemName` (required) — the System asset name used to stitch harvested + lineage to catalog assets. Must match the system under which jdbc-ingestion registered + the database's assets. +- `databaseNames` (list; **required in BOTH modes**, even though the manifest marks it + optional) — the databases to harvest. The harvester fails with "No database names + provided" when it is empty, and Edge accepts the capability without it — the mistake + only surfaces later, in the failed harvest job. Always collect the names from the + user directly; leave the manifest's `databaseNamesJson` file-upload variant unset + (there are no files in this flow). +- `schemaNames` (list, optional) — restrict harvesting to these schemas. +- `snowflakeDays` (default `365`; `SQL-API` mode) — how many days of `ACCESS_HISTORY` + to read. Lower it for faster harvests on busy accounts. +- `extraDatabaseDefinitions` (list, `SQL-API` only) — databases to load metadata for + without harvesting lineage from them (targets of cross-database references). + +## Queries — defaults are automatic; only overrides are a conversation + +Each query parameter is an editable SQL statement with a built-in default. **Never +collect query SQL from the user unprompted, and never copy the manifest's default SQL +into `parameters`.** "Keep the default" is expressed one of two ways: + +- **omit the parameter** — Edge materializes the manifest default at save time (the + create response echoes the filled queries back; that is expected, not an error), or +- pass the literal string **`"use-default"`** — the sentinel the Edge UI stores; the + harvester substitutes its built-in query at run time. + +Only when the user explicitly wants to customize a harvest query, offer the chosen +mode's query types and set that one parameter to the user's SQL: + +- **`SQL` mode:** `columns`, `views`, `procedures`; `external_stages` (optional). +- **`SQL-API` mode:** `access_history` (the core lineage query of this mode), + `object_dependencies`, `columns_joined`. +- **Either mode (optional):** `dynamic_tables`, `semantic_views`. + +Placeholders like `##DBNAME##`, `##DBNAMES##`, `##SCHEMANAMES##`, `##DAYS##` are +substituted at run time and must be preserved in any override. + +## Custom Properties — suggest these proactively + +The Custom Properties group is a **single parameter, `customParameters`** — a list of +key/value objects the harvester reads at run time. `techlinHost` and `techlinKey` are +**not top-level parameters**: set at the top level of `parameters` they are silently +ignored. The encoding: + +```json +"customParameters": [ + {"name": "techlinHost", "value": "https://techlin-.example.com", + "type": "string", "secret": false, "encrypted": false, "fromVault": false}, + {"name": "techlinKey", "value": "", + "type": "string", "secret": false, "encrypted": false, "fromVault": false} +] +``` + +- `techlinHost` — host URL of the Collibra Data Lineage (techlin) server the harvested + batches are uploaded to. +- `techlinKey` — the API key for that server. + +The values are tenant-specific (the lineage server assigned to the customer's +environment). They are optional — but always ask for them explicitly; deferring them +is the **user's** decision, never a silent default. If the user doesn't have the +values at hand, create without them and point out they can be added later via +`edge_create_capability` with the `capabilityId` (upsert) — see "Updating an existing +capability" in the parent `SKILL.md`. + +## Advanced Properties + +- `processingLevel` — `Load` (harvest only, raw metadata inspectable), `Analyze` + (harvest + analyze), `Sync` (default: also synchronize lineage into the catalog). +- `dependentSourceIds` — Source IDs of other lineage sources this one references, so + cross-source lineage resolves. +- `databaseSystemMapping` — map database names to system names when one capability + covers databases that belong to different systems. +- `deleteRawMetadataAfterProcessing` — remove harvested raw metadata after processing. +- `active` — set to `false` and re-run to **remove** this source's lineage from the + lineage server. Edge fills `active: true` at creation; there is nothing to ask. +- `techlinAdminConnection` (beta) — optional connection to the lineage (TechLin) server + admin API. Leave unset unless the user asks for it. + +## Gotchas + +- **Stitching needs ingested assets.** Lineage attaches to the Schema/Table/Column + assets created by jdbc-ingestion under `collibraSystemName`; if ingestion never ran + (or ran under a different system name), the harvest succeeds but lineage doesn't link + to catalog assets. +- **The trigger is asset-keyed.** `start_technical_lineage` takes the Database asset + UUID; DGC finds the matching Technical Lineage capability itself. Passing the + capability id fails. +- **Edge fills required defaults at save.** The `edge_create_capability` response + comes back with the required query parameters and `active: "true"` materialized — + expected behavior, not something to re-collect or re-send. +- **One capability per (type, connection).** `edge_create_capability` fails with + `400 — "Connection(s) […] already used in the capability of the same type"` when a + Technical Lineage capability already exists for that connection. Treat it as + "already exists": go back to `edge_find_capabilities` and the update or trigger + path — never retry with a different connection. +- **Requiredness comes from this reference, not the manifest.** The manifest marks + `databaseNames` optional (fatal at runtime when missing) and marks both modes' + query parameters required (harmless — Edge auto-fills them). If the live manifest + lists a parameter this reference doesn't mention, surface it to the user instead of + guessing. \ No newline at end of file diff --git a/pkg/tools/edge_create_capability/tool.go b/pkg/tools/edge_create_capability/tool.go index d3610e2..72e91b7 100644 --- a/pkg/tools/edge_create_capability/tool.go +++ b/pkg/tools/edge_create_capability/tool.go @@ -20,7 +20,7 @@ type Input struct { Description string `json:"description,omitempty" jsonschema:"Optional description of the capability."` TypeID string `json:"typeId" jsonschema:"The id of the capability type (e.g. 'jdbc-ingestion'). Use the edge_list_capability_types tool to discover available types and their expected parameters."` EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this capability will run. Use the edge_list_sites tool to discover available sites."` - Parameters map[string]any `json:"parameters" jsonschema:"Capability install parameters as defined by the capability type's manifest. For jdbc-ingestion, this includes 'connection' (the id of a connection created via edge_create_connection), 'data-source-type', 'message-mode', and an optional 'other-settings' list of {name, type, value}."` + Parameters map[string]any `json:"parameters" jsonschema:"Capability install parameters as defined by the capability type's manifest — read it via edge_list_capability_types (pass a query) and ask the user for user-choice values; never invent them. For jdbc-ingestion this includes 'connection' (the id of a connection created via edge_create_connection), 'data-source-type', 'message-mode', and an optional 'other-settings' list of {name, type, value}. For technical lineage capabilities (edgeharvester-*) the parameters are the capability's entire configuration — collect and confirm them with the user before calling this tool (see the collibra/techlin skill); custom properties like techlinHost/techlinKey go inside the 'customParameters' list parameter as {name, value, type: 'string', secret: false, encrypted: false, fromVault: false} objects (never top-level); harvest-query parameters are omitted to accept the defaults (the server fills required ones) or set to the literal string 'use-default' — never invented SQL."` } type Output struct { @@ -33,7 +33,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "edge_create_capability", Title: "Create or Update Edge Capability", - Description: "Creates or updates an Edge capability (e.g. jdbc-ingestion) via the private Edge capability management API. Does not run the capability — use start_ingestion to trigger a jdbc-ingestion run.", + Description: "Creates or updates an Edge capability (e.g. jdbc-ingestion or a technical lineage capability) via the private Edge capability management API. Parameters must come from the capability type's manifest (edge_list_capability_types) and, for user-choice values, from the user — confirm the full set before calling. The server materializes defaults for required manifest parameters at save (the response echoes them back), and rejects a second capability of the same type on one connection (400 'already used') — treat that as 'the capability already exists', not as a reason to switch connections. Does not run the capability — use start_ingestion to trigger a jdbc-ingestion run, or start_technical_lineage for a technical lineage harvest.", Handler: handler(collibraClient), Permissions: []string{"dgc.edge-integration-capability-manage"}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/edge_list_capability_types/tool.go b/pkg/tools/edge_list_capability_types/tool.go index 5798e72..8f749ef 100644 --- a/pkg/tools/edge_list_capability_types/tool.go +++ b/pkg/tools/edge_list_capability_types/tool.go @@ -20,7 +20,7 @@ type Input struct { } type Output struct { - CapabilityTypes []clients.CapabilityType `json:"capabilityTypes" jsonschema:"Capability types available on this edge site (e.g. 'jdbc-ingestion'). Manifest is only populated when query narrows the results — see query's description."` + CapabilityTypes []clients.CapabilityType `json:"capabilityTypes" jsonschema:"Capability types available on this edge site (e.g. 'jdbc-ingestion'). Manifest is only populated when query narrows the results — see query's description. The manifest's 'required'/'default' flags describe the Edge UI form, not runtime validation — the per-source skill references (e.g. collibra/techlin) carry the runtime requiredness."` ConnectionTypes []clients.ConnectionType `json:"connectionTypes" jsonschema:"Connection types available on this edge site (e.g. 'Generic' or a vendor-specific type). Manifest is only populated when query narrows the results — see query's description."` } From e41579748317aea231859e472ce1d89a221b9215 Mon Sep 17 00:00:00 2001 From: Yauheni Hlazkou Date: Fri, 10 Jul 2026 14:19:30 +0200 Subject: [PATCH 6/6] =?UTF-8?q?feat:=20secret-safe=20techlinKey=20flow=20?= =?UTF-8?q?=E2=80=94=20collect=20host=20only,=20key=20added=20via=20Edge?= =?UTF-8?q?=20UI,=20trigger=20gated=20on=20confirmation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit techlinKey passed through edge_create_capability is stored plaintext and readable back through the management API, so the key must never transit the conversation. When the user opts into configuring the techlin server: - collect techlinHost only and create the capability without the key - the user then adds techlinKey in the Collibra/Edge UI (capability Custom Properties, marked secret so it is masked and encrypted) - start_technical_lineage is triggered only after the user confirms the key is in place - techlinHost alone can still be changed later via the upsert; the key always goes in through the UI Encoded in the snowflake reference (question script + custom-properties flow), the techlin skill (conversation rule, trigger step gate, hard rule 5), and the edge_create_capability parameters description so the rule also holds for skills-off clients. --- pkg/skills/files/collibra/techlin/SKILL.md | 30 +++++++---- .../collibra/techlin/references/snowflake.md | 50 ++++++++++++------- pkg/tools/edge_create_capability/tool.go | 2 +- 3 files changed, 51 insertions(+), 31 deletions(-) diff --git a/pkg/skills/files/collibra/techlin/SKILL.md b/pkg/skills/files/collibra/techlin/SKILL.md index 1413855..ed38779 100644 --- a/pkg/skills/files/collibra/techlin/SKILL.md +++ b/pkg/skills/files/collibra/techlin/SKILL.md @@ -117,6 +117,8 @@ create the Database asset any other way. See "Capability parameters". 7. **`start_technical_lineage`** with the Database asset UUID from step 1 — **not** the capability id. Only reached once step 3 (or step 6) confirmed the capability exists. + If the user opted into the techlin server configuration in step 6, wait for their + confirmation that `techlinKey` was added in the Edge UI before triggering. Success means *submitted*, nothing more. 8. **Track:** the trigger returns no job id. Find the spawned DGC job with **`jobs_find`** (jobs started at/after the trigger), then poll it with **`get_job_status`** until it @@ -181,13 +183,17 @@ Ordering and rules for the conversation: capabilities via `edge_list_capabilities`/`edge_find_capabilities`), with the `collibraSystemName`, or with a database/schema name. A colliding Source ID silently overwrites or cross-links another source's lineage. -4. **Proactively offer the `techlinHost` and `techlinKey` custom properties** — they - point the capability at the Collibra Data Lineage (techlin) server and are passed - **inside the `customParameters` list parameter** (the per-source reference shows - the exact encoding), never as top-level parameters. They are optional: ask for - them explicitly; if the user doesn't have the values at hand, deferring them is - the **user's** decision, never a silent default — they can be added later (see - "Updating an existing capability"). +4. **Proactively offer the techlin server configuration** (`techlinHost` / + `techlinKey` custom properties) — they point the capability at the Collibra Data + Lineage (techlin) server and are passed **inside the `customParameters` list + parameter** (the per-source reference shows the exact encoding), never as + top-level parameters. They are optional, and the two are handled differently: + when the user wants them, collect **`techlinHost` only** and create the + capability without the key — **`techlinKey` is a secret and is never collected, + stored, or echoed in the conversation**. After creation, the user adds + `techlinKey` in the Collibra/Edge UI (Custom Properties, marked secret); + **trigger the harvest only after the user confirms the key is in place**. + Declining both is the user's decision, never a silent default. 5. **Confirm the complete set.** Present every parameter — chosen values, applied defaults, custom properties — in one table and get an explicit confirmation before calling `edge_create_capability`. @@ -231,10 +237,12 @@ rule as creation — show what will change and get a go-ahead first. parameters the reference doesn't mention are surfaced to the user, not guessed. 5. **Ask the user for parameter values and confirm the full set** before `edge_create_capability` — even when the user already approved creating the - capability; approval covers the creation, not skipping the questions. Never ask for - credentials. The `connection` value and the `edgeSiteId` are not user choices — - take both from `get_registered_database`; the capability lives on the same edge - site as the connection. + capability; approval covers the creation, not skipping the questions. Never ask + for credentials, and never collect the `techlinKey` value in the conversation — + the user adds it in the Edge UI after creation (see the per-source reference). + The `connection` value and the `edgeSiteId` are not user choices — take both from + `get_registered_database`; the capability lives on the same edge site as the + connection. 6. **Only `start_technical_lineage` triggers a harvest** — never `edge_run_capability` and never `catalog_etl_start_job`. Pass the **Database asset** UUID, not the capability id. diff --git a/pkg/skills/files/collibra/techlin/references/snowflake.md b/pkg/skills/files/collibra/techlin/references/snowflake.md index e67804d..c957d14 100644 --- a/pkg/skills/files/collibra/techlin/references/snowflake.md +++ b/pkg/skills/files/collibra/techlin/references/snowflake.md @@ -43,8 +43,10 @@ This is the question script for the parameter checkpoint (step 6 of the parent database's assets under. 5. **Queries** — ask only whether the user wants to *override* any harvest query (almost nobody does; defaults apply automatically — see "Queries"). -6. **Custom Properties** — offer `techlinHost` + `techlinKey` (optional; note the - `customParameters` encoding below). +6. **Techlin server (Custom Properties)** — ask whether the user wants to configure + the Collibra Data Lineage (techlin) server for this capability. If yes: collect + **`techlinHost` only** — never the key. `techlinKey` is a secret; the user adds + it after creation, outside this conversation (see "Custom Properties" below). 7. **Advanced Properties** — only when the user asks for them. 8. **Present the complete parameter set in one table and get an explicit "create it"** before calling `edge_create_capability`. @@ -113,32 +115,42 @@ mode's query types and set that one parameter to the user's SQL: Placeholders like `##DBNAME##`, `##DBNAMES##`, `##SCHEMANAMES##`, `##DAYS##` are substituted at run time and must be preserved in any override. -## Custom Properties — suggest these proactively +## Custom Properties — the techlin server; the key never goes through the chat The Custom Properties group is a **single parameter, `customParameters`** — a list of key/value objects the harvester reads at run time. `techlinHost` and `techlinKey` are **not top-level parameters**: set at the top level of `parameters` they are silently -ignored. The encoding: - -```json -"customParameters": [ - {"name": "techlinHost", "value": "https://techlin-.example.com", - "type": "string", "secret": false, "encrypted": false, "fromVault": false}, - {"name": "techlinKey", "value": "", - "type": "string", "secret": false, "encrypted": false, "fromVault": false} -] -``` +ignored. - `techlinHost` — host URL of the Collibra Data Lineage (techlin) server the harvested batches are uploaded to. -- `techlinKey` — the API key for that server. +- `techlinKey` — the API key for that server. **A secret: never collect, store, or + echo its value in the conversation** — a value passed through + `edge_create_capability` is stored plaintext and readable back through the API. The values are tenant-specific (the lineage server assigned to the customer's -environment). They are optional — but always ask for them explicitly; deferring them -is the **user's** decision, never a silent default. If the user doesn't have the -values at hand, create without them and point out they can be added later via -`edge_create_capability` with the `capabilityId` (upsert) — see "Updating an existing -capability" in the parent `SKILL.md`. +environment) and optional — always ask explicitly whether the user wants to configure +them; declining is the **user's** decision, never a silent default. When the user +wants them, the flow is split: + +1. Ask for **`techlinHost` only** and include it in the capability: + + ```json + {"customParameters": [ + {"name": "techlinHost", "value": "https://techlin-.example.com", + "type": "string", "secret": false, "encrypted": false, "fromVault": false} + ]} + ``` + +2. Create the capability with the confirmed parameter set — **without `techlinKey`**. +3. Tell the user to add `techlinKey` themselves in the Collibra/Edge UI — the + capability's **Custom Properties** group → add property `techlinKey`, **marked + secret** so it is masked and encrypted — and to say when it is done. +4. **Trigger the harvest only after the user confirms the key is in place.** + +`techlinHost` alone can also be added or changed later via the upsert (see "Updating +an existing capability" in the parent `SKILL.md`); `techlinKey` always goes in +through the Edge UI. ## Advanced Properties diff --git a/pkg/tools/edge_create_capability/tool.go b/pkg/tools/edge_create_capability/tool.go index 72e91b7..ebec22f 100644 --- a/pkg/tools/edge_create_capability/tool.go +++ b/pkg/tools/edge_create_capability/tool.go @@ -20,7 +20,7 @@ type Input struct { Description string `json:"description,omitempty" jsonschema:"Optional description of the capability."` TypeID string `json:"typeId" jsonschema:"The id of the capability type (e.g. 'jdbc-ingestion'). Use the edge_list_capability_types tool to discover available types and their expected parameters."` EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this capability will run. Use the edge_list_sites tool to discover available sites."` - Parameters map[string]any `json:"parameters" jsonschema:"Capability install parameters as defined by the capability type's manifest — read it via edge_list_capability_types (pass a query) and ask the user for user-choice values; never invent them. For jdbc-ingestion this includes 'connection' (the id of a connection created via edge_create_connection), 'data-source-type', 'message-mode', and an optional 'other-settings' list of {name, type, value}. For technical lineage capabilities (edgeharvester-*) the parameters are the capability's entire configuration — collect and confirm them with the user before calling this tool (see the collibra/techlin skill); custom properties like techlinHost/techlinKey go inside the 'customParameters' list parameter as {name, value, type: 'string', secret: false, encrypted: false, fromVault: false} objects (never top-level); harvest-query parameters are omitted to accept the defaults (the server fills required ones) or set to the literal string 'use-default' — never invented SQL."` + Parameters map[string]any `json:"parameters" jsonschema:"Capability install parameters as defined by the capability type's manifest — read it via edge_list_capability_types (pass a query) and ask the user for user-choice values; never invent them. For jdbc-ingestion this includes 'connection' (the id of a connection created via edge_create_connection), 'data-source-type', 'message-mode', and an optional 'other-settings' list of {name, type, value}. For technical lineage capabilities (edgeharvester-*) the parameters are the capability's entire configuration — collect and confirm them with the user before calling this tool (see the collibra/techlin skill); custom properties like techlinHost go inside the 'customParameters' list parameter as {name, value, type: 'string', secret: false, encrypted: false, fromVault: false} objects (never top-level); the techlinKey value is a secret — never collect it in conversation or pass it here; the user adds it in the Collibra/Edge UI after creation (see the collibra/techlin skill); harvest-query parameters are omitted to accept the defaults (the server fills required ones) or set to the literal string 'use-default' — never invented SQL."` } type Output struct {