diff --git a/.gitignore b/.gitignore index 50fff47..2136701 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build mcp.yaml oas/** mise.toml +.DS_Store diff --git a/README.md b/README.md index ba6d0df..d75dc36 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,15 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`discover_business_glossary`](pkg/tools/discover_business_glossary/) - Ask questions about terms and definitions. Note that this tool leverages Collibra AI and therefore consumes Collibra Units (CUs). **Requires:** `dgc.ai-copilot` - [`discover_data_assets`](pkg/tools/discover_data_assets/) - Query available data assets using natural language. Note that this tool leverages Collibra AI and therefore consumes Collibra Units (CUs). **Requires:** `dgc.ai-copilot` +- [`edge_find_connections`](pkg/tools/edge_find_connections/) - Find existing Edge connections by name, to reuse instead of duplicating. **Requires:** `dgc.edge-view-connections-and-capabilities` +- [`find_domain_types`](pkg/tools/find_domain_types/) - Resolve a domain type name (e.g. "Physical Data Dictionary") to its UUID +- [`find_users`](pkg/tools/find_users/) - Resolve a user name to their UUID, e.g. for a database asset's owners - [`get_asset_details`](pkg/tools/get_asset_details/) - Retrieve detailed information about specific assets by UUID, including the asset's assignable attribute schema (every attribute it can hold, including empty ones) - [`get_business_term_data`](pkg/tools/get_business_term_data/) - Trace a business term back to its connected physical data assets +- [`get_job_status`](pkg/tools/get_job_status/) - Poll the status of a DGC job, e.g. a jdbc-ingestion sync started by `start_ingestion` - [`get_column_semantics`](pkg/tools/get_column_semantics/) - Retrieve data attributes, measures, and business assets connected to a column +- [`get_data_source_setup_guide`](pkg/tools/get_data_source_setup_guide/) - Look up JDBC driver class, connection string format, and connection properties for a data source from Collibra's documentation +- [`edge_get_job_status`](pkg/tools/edge_get_job_status/) - Poll the status of an Edge job, e.g. a connection test or capability run - [`get_lineage_downstream`](pkg/tools/get_lineage_downstream/) - Get downstream technical lineage (consumers) for a data entity - [`get_lineage_entity`](pkg/tools/get_lineage_entity/) - Get metadata about a specific entity in the technical lineage graph - [`get_lineage_transformation`](pkg/tools/get_lineage_transformation/) - Get details and logic of a specific data transformation @@ -22,7 +28,9 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`get_measure_data`](pkg/tools/get_measure_data/) - Trace a measure back to its underlying physical columns and tables - [`get_table_semantics`](pkg/tools/get_table_semantics/) - Retrieve the semantic layer for a table: columns, data attributes, and connected measures - [`list_asset_types`](pkg/tools/list_asset_types/) - List available asset types +- [`edge_list_capability_types`](pkg/tools/edge_list_capability_types/) - List capability and connection types available on an Edge site, with their manifests. **Requires:** `dgc.edge-view-connections-and-capabilities` - [`list_data_contract`](pkg/tools/list_data_contracts/) - List data contracts with pagination +- [`edge_list_sites`](pkg/tools/edge_list_sites/) - List available Edge sites. **Requires:** `dgc.edge-view-connections-and-capabilities` - [`prepare_create_asset`](pkg/tools/prepare_create_asset/) - Read-only companion to `create_asset`: enumerate available asset types and domains, resolve a UUID/publicId/displayName for either, and hydrate the scoped attribute and relation schema for a chosen pair - [`pull_data_contract_manifest`](pkg/tools/pull_data_contract_manifest/) - Download manifest for a data contract - [`search_asset_keyword`](pkg/tools/search_asset_keyword/) - Wildcard keyword search for assets; filters (status, community, domain, domain type, asset type, created-by) accept names or UUIDs @@ -30,11 +38,17 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`search_data_classification_match`](pkg/tools/search_data_classification_matches/) - Search for associations between data classes and assets. **Requires:** `dgc.classify`, `dgc.catalog` - [`search_lineage_entities`](pkg/tools/search_lineage_entities/) - Search for entities in the technical lineage graph - [`search_lineage_transformations`](pkg/tools/search_lineage_transformations/) - Search for transformations in the technical lineage graph +- [`test_connection`](pkg/tools/test_connection/) - Test whether an Edge connection can reach its data source. **Requires:** `dgc.edge-integration-capability-manage` ### Write Tools - [`add_data_classification_match`](pkg/tools/add_data_classification_match/) - Associate a data class with an asset. **Requires:** `dgc.classify`, `dgc.catalog` +- [`configure_database_schemas`](pkg/tools/configure_database_schemas/) - Discover and configure schema/table synchronization rules for a database registered via register_database, for jdbc-ingestion - [`create_asset`](pkg/tools/create_asset/) - Create a new asset of any type. Resolves `assetType` (UUID, publicId, or display name), `domain` (UUID or name), `status` (UUID or name), and attributes (by name or typeId) server-side; converts Markdown to HTML for `RICH_TEXT` attributes; gates on duplicate-name (default `allowDuplicate: false`) +- [`edge_create_capability`](pkg/tools/edge_create_capability/) - Create or update an Edge capability (e.g. jdbc-ingestion). **Requires:** `dgc.edge-integration-capability-manage` +- [`create_community`](pkg/tools/create_community/) - Create a DGC community +- [`edge_create_connection`](pkg/tools/edge_create_connection/) - Create or update an Edge connection (e.g. a JDBC connection). **Requires:** `dgc.edge-integration-capability-manage` +- [`create_domain`](pkg/tools/create_domain/) - Create a DGC domain within a community - [`edit_asset`](pkg/tools/edit_asset/) - Edit an existing asset via a list of typed operations: - `set_attribute`, `add_attribute`, `remove_attribute` - set an attribute value (creates if empty, updates if present), append an extra value to a multi-valued attribute, or clear one (e.g. `Definition`, `Note`) - `update_property` - rename the asset (`name`), change its `displayName`, or change its `statusId` (status name or UUID accepted) @@ -44,7 +58,10 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - `remove_responsibility` - unassign a user or group from a resource role (only directly-assigned responsibilities, not inherited ones) - [`init_data_contract`](pkg/tools/init_data_contract/) - Initialize a new data contract asset governing a Data Product Port, with an optional initial manifest. **Requires:** `dgc.data-contract` - [`push_data_contract_manifest`](pkg/tools/push_data_contract_manifest/) - Upload manifest for a data contract. **Requires:** `dgc.data-contract` +- [`register_database`](pkg/tools/register_database/) - Discover a database through an Edge connection and register it as a Database asset, for jdbc-ingestion - [`remove_data_classification_match`](pkg/tools/remove_data_classification_match/) - Remove a classification match. **Requires:** `dgc.classify`, `dgc.catalog`, `dgc.data-classes-edit` +- [`start_ingestion`](pkg/tools/start_ingestion/) - Trigger a jdbc-ingestion capability run for a registered database +- [`upload_file`](pkg/tools/upload_file/) - Upload a file (JDBC driver, TLS certificate, private key, keytab, …) to an Edge site. **Requires:** `dgc.edge-integration-capability-manage` ## Quick Start @@ -213,7 +230,7 @@ Some functionality ships behind an opt-in `experimental` flag. These features ar - `context-specifications` — Context specification tools: `list_context_specifications`, `get_context_specification`, and the `contextSpecificationId` parameter on `get_asset_details`. These tools generate structured YAML context for assets using the Semantic Blueprint API. -- `skills` — Embedded skill catalog served via two additional tools, `list_collibra_skills` and `load_collibra_skill`. Skills are short Markdown guides that document multi-step Collibra workflows (discovery, lineage, asset create/edit, …) for the connecting LLM. See [SKILLS.md](SKILLS.md) for the catalog. +- `skills` — Embedded skill catalog served via two additional tools, `list_collibra_skills` and `load_collibra_skill`. Skills are short Markdown guides that document multi-step Collibra workflows (discovery, lineage, asset create/edit, jdbc-ingestion setup, …) for the connecting LLM. See [SKILLS.md](SKILLS.md) for the catalog. Point chip at an **external skills directory** with `--skills-dir=` (or `COLLIBRA_MCP_SKILLS_DIR`, or `mcp.skills-dir` in YAML) to add your own skills on top of the embedded ones. The expected layout is `///SKILL.md` (with optional `references/*.md` and `_shared/*.md` siblings) — same as the bundled catalog. External skills whose name matches an embedded skill (e.g. `collibra/lineage`) **fully replace** the embedded entry, including its resources, so you can override the shipped guides without rebuilding chip. `~` and `~user` in the path are expanded. diff --git a/SKILLS.md b/SKILLS.md index 2c9e872..2fbc277 100644 --- a/SKILLS.md +++ b/SKILLS.md @@ -17,6 +17,8 @@ those tools. Skill content lives in [`pkg/skills/files/collibra/`](pkg/skills/fi | `collibra/lineage` | Technical lineage; DGC UUID ↔ lineage entity ID bridge; column-level workaround | | `collibra/asset-create` | `create_asset` workflow; RICH_TEXT Markdown handling; duplicate gating | | `collibra/asset-edit` | `edit_asset` operation types | +| `collibra/jdbc-ingestion` | End-to-end jdbc-ingestion setup: Edge connection/capability, database registration, running and polling a sync | +| `collibra/etl-integration` | End-to-end ETL sync integrations (Dataplex, Databricks UC, Purview, Sigma, …): connection, capability, generic config (fetched live), schedule/run; per-integration references | Each skill is one `SKILL.md` per directory, with frontmatter (`description`, `related`) and an optional `references/` directory for bundled reference documents. diff --git a/cmd/chip/config.go b/cmd/chip/config.go index 039c1fd..62f48e4 100644 --- a/cmd/chip/config.go +++ b/cmd/chip/config.go @@ -94,6 +94,11 @@ func initConfigOptions() { _ = viper.BindPFlag("mcp.enable-debug-tools", pflag.Lookup("enable-debug-tools")) viper.SetDefault("mcp.enable-debug-tools", false) + pflag.Bool("allow-local-file-upload", false, "Allow upload_file to read files directly from this machine's local disk (filePath input); off by default. Only meaningful when running chip locally against your own files — never enable this for a shared/remote deployment (env: COLLIBRA_MCP_ALLOW_LOCAL_FILE_UPLOAD)") + _ = viper.BindEnv("mcp.allow-local-file-upload", "COLLIBRA_MCP_ALLOW_LOCAL_FILE_UPLOAD") + _ = viper.BindPFlag("mcp.allow-local-file-upload", pflag.Lookup("allow-local-file-upload")) + viper.SetDefault("mcp.allow-local-file-upload", false) + pflag.StringSlice("experimental", []string{}, "Comma-separated list of opt-in experimental features to enable (env: COLLIBRA_MCP_EXPERIMENTAL). See EXPERIMENTAL FEATURES below for valid names.") _ = viper.BindEnv("mcp.experimental", "COLLIBRA_MCP_EXPERIMENTAL") _ = viper.BindPFlag("mcp.experimental", pflag.Lookup("experimental")) @@ -128,6 +133,7 @@ ENVIRONMENT VARIABLES: COLLIBRA_MCP_ENABLED_TOOLS Optional comma-separated list of tool names to enable instead of enabling all tools, cannot be used with disabled-tools COLLIBRA_MCP_DISABLED_TOOLS Optional comma-separated list of tool names to disable while enabling the remaining tools, cannot be used with enabled-tools COLLIBRA_MCP_ENABLE_DEBUG_TOOLS Enable debug tools (default: false) + COLLIBRA_MCP_ALLOW_LOCAL_FILE_UPLOAD Allow upload_file to read files from local disk (default: false); local/personal use only, never for a shared deployment COLLIBRA_MCP_EXPERIMENTAL Comma-separated list of opt-in experimental features to enable (see EXPERIMENTAL FEATURES below) COLLIBRA_MCP_SKILLS_DIR Optional path to an external skills directory merged on top of the embedded catalog (requires the 'skills' experimental feature) @@ -162,6 +168,7 @@ CONFIGURATION FILE EXAMPLE: # - "tool3" # - "tool4" enable-debug-tools: false # Optional: enable debug tools (default: false) + allow-local-file-upload: false # Optional: allow upload_file to read local files (default: false; local use only) # experimental: # Optional: opt-in experimental features (off by default) # - "skills" # skills-dir: "/path/to/skills" # Optional: external skills dir (requires the 'skills' experimental feature) @@ -218,14 +225,15 @@ type CollibraApiConfig struct { // ServerConfig holds server configuration type McpConfig struct { - Mode string `mapstructure:"mode"` // "stdio", "http", "http-sse", or "http-streamable" - Http HttpConfig `mapstructure:"http"` - Stdio StdioConfig `mapstructure:"stdio"` - EnabledTools []string `mapstructure:"enabled-tools"` - DisabledTools []string `mapstructure:"disabled-tools"` - EnableDebugTools bool `mapstructure:"enable-debug-tools"` - Experimental []string `mapstructure:"experimental"` - SkillsDir string `mapstructure:"skills-dir"` + Mode string `mapstructure:"mode"` // "stdio", "http", "http-sse", or "http-streamable" + Http HttpConfig `mapstructure:"http"` + Stdio StdioConfig `mapstructure:"stdio"` + EnabledTools []string `mapstructure:"enabled-tools"` + DisabledTools []string `mapstructure:"disabled-tools"` + EnableDebugTools bool `mapstructure:"enable-debug-tools"` + AllowLocalFileUpload bool `mapstructure:"allow-local-file-upload"` + Experimental []string `mapstructure:"experimental"` + SkillsDir string `mapstructure:"skills-dir"` } type HttpConfig struct { diff --git a/cmd/chip/experimental.go b/cmd/chip/experimental.go index faf0a22..874f06f 100644 --- a/cmd/chip/experimental.go +++ b/cmd/chip/experimental.go @@ -17,7 +17,7 @@ import ( // the help text, example YAML, and validation warning all read from this // map so no other code needs to change. var knownExperimentalFeatures = map[string]string{ - skills.FeatureName: "Embedded skill catalog served via list_collibra_skills and load_collibra_skill.", + skills.FeatureName: "Embedded skill catalog served via list_collibra_skills and load_collibra_skill.", tools.ContextSpecificationsFeature: "Context specification tools: list_context_specifications, get_context_specification, and contextSpecificationId parameter on get_asset_details.", } diff --git a/cmd/chip/main.go b/cmd/chip/main.go index 8e4856a..331f138 100644 --- a/cmd/chip/main.go +++ b/cmd/chip/main.go @@ -31,11 +31,12 @@ func main() { client := newCollibraClient(config) toolConfig := &chip.ServerToolConfig{ - EnabledTools: config.Mcp.EnabledTools, - DisabledTools: config.Mcp.DisabledTools, - EnableDebugTools: config.Mcp.EnableDebugTools, - Experimental: config.Mcp.Experimental, - SkillsDir: config.Mcp.SkillsDir, + EnabledTools: config.Mcp.EnabledTools, + DisabledTools: config.Mcp.DisabledTools, + EnableDebugTools: config.Mcp.EnableDebugTools, + AllowLocalFileUpload: config.Mcp.AllowLocalFileUpload, + Experimental: config.Mcp.Experimental, + SkillsDir: config.Mcp.SkillsDir, } serverOpts := []chip.ServerOption{ diff --git a/go.mod b/go.mod index 50af924..666a8db 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/yuin/goldmark v1.8.2 + golang.org/x/net v0.56.0 ) require ( @@ -26,7 +27,7 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 81fa5d8..155aa8d 100644 --- a/go.sum +++ b/go.sum @@ -56,14 +56,16 @@ github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= -golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= -golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/job-api.yaml b/job-api.yaml new file mode 100644 index 0000000..d781a61 --- /dev/null +++ b/job-api.yaml @@ -0,0 +1,777 @@ +--- +openapi: 3.0.3 +info: + description: This API provides the ability to retrieve job progress. + title: Collibra Job API + version: 1.0.0 + x-audience: public +servers: +- url: /rest/jobs/v1 +security: +- basicAuth: [] +- jwtAuth: [] +tags: +- description: Job endpoints provide the ability to retrieve job progress. + name: Jobs +paths: + /jobs: + get: + description: Returns jobs matching the given search criteria. Only parameters + that are specified in this request and have not null values are + used for filtering. All other parameters are ignored. + operationId: findJobs + parameters: + - description: "The match mode used to compare name. If the match\ + \ mode is EXACT the search is case-sensitive, otherwise the\ + \ search is case-insensitive." + explode: true + in: query + name: nameMatchMode + required: false + schema: + default: ANYWHERE + enum: + - START + - END + - ANYWHERE + - EXACT + example: EXACT + type: string + style: form + - description: The name of the job to search for. + explode: true + in: query + name: name + required: false + schema: + example: Sample job + type: string + style: form + - description: The list of results which the jobs should be searched for. + explode: true + in: query + name: result + required: false + schema: + items: + enum: + - NOT_SET + - SUCCESS + - COMPLETED_WITH_ERROR + - FAILURE + - ABORTED + example: SUCCESS + type: string + type: array + style: form + - description: The list of states which the jobs should be searched for. + explode: true + in: query + name: state + required: false + schema: + items: + enum: + - WAITING + - RUNNING + - COMPLETED + - FAILED + - DELETED + example: COMPLETED + type: string + type: array + style: form + - description: The list of types of the jobs. + explode: true + in: query + name: type + required: false + schema: + items: + example: IMPORT + type: string + type: array + style: form + - description: The ID of the user who created the searched job. + explode: true + in: query + name: user + required: false + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: form + - description: "The field that should be used as reference for sorting, \nin\ + \ addition to the CREATED_ON field in DESC order.\n" + explode: true + in: query + name: sortField + required: false + schema: + enum: + - NAME + - TYPE + - START_DATE + - END_DATE + - STATE + - RESULT + example: NAME + type: string + style: form + - description: The order of sorting. + explode: true + in: query + name: sortOrder + required: false + schema: + default: DESC + enum: + - ASC + - DESC + example: ASC + type: string + style: form + - description: "The cursor pointing to the first resource to be included in\ + \ the response. \nThis cursor cannot be created and must have been extracted\ + \ from a response returned by a previous API call.\n\nIf this parameter\ + \ is missing, the API returns the resources starting from the first available\ + \ resource, at index `0`.\n" + explode: true + in: query + name: cursor + required: false + schema: + type: string + style: form + - description: | + The maximum number of resources to retrieve. + + If not set, the default limit of `10` is be used. The maximum value for this parameter is `50`. + explode: true + in: query + name: pageSize + required: false + schema: + default: 10 + maximum: 50 + minimum: 1 + type: integer + style: form + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/JobPagedResponse" + description: The paged response with jobs matching filter criteria + "400": + content: + application/json: + example: + userMessage: The request is not valid. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The request is not valid. + "401": + content: + application/json: + example: + userMessage: Failed to authenticate the request through basic credentials. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The client is not authenticated. + "500": + content: + application/json: + example: + userMessage: Internal server error. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Internal server error. + summary: Find jobs + tags: + - Jobs + /jobs/{jobId}: + delete: + description: | + Soft deletes the Job identified by the given UUID. + Users can delete their own jobs and Admins can delete any job. + operationId: deleteJob + parameters: + - description: The ID of the job. + explode: false + in: path + name: jobId + required: true + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: simple + - description: An optional message for the job deletion. + explode: true + in: query + name: message + required: false + schema: + default: Deleted by user + maxLength: 128 + type: string + style: form + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/JobDelete" + description: | + Use the message query parameter instead. Deprecated, will be removed in 2026.06. + Note: The message defined in the query parameter takes precedence over the request body. + required: false + responses: + "204": + description: Job successfully deleted. + "400": + content: + application/json: + example: + userMessage: The request is not valid. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The request is not valid. + "401": + content: + application/json: + example: + userMessage: Failed to authenticate the request through basic credentials. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The client is not authenticated. + "404": + content: + application/json: + example: + userMessage: The resource could not be found. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Resource not found. + "500": + content: + application/json: + example: + userMessage: Internal server error. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Internal server error. + security: + - basicAuth: [] + - jwtAuth: [] + summary: Soft delete Job. + tags: + - Jobs + get: + description: Retrieves the Job identified by the given UUID. + operationId: getJob + parameters: + - description: The ID of the job. + explode: false + in: path + name: jobId + required: true + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: simple + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Job" + description: Job found. + "400": + content: + application/json: + example: + userMessage: The request is not valid. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The request is not valid. + "401": + content: + application/json: + example: + userMessage: Failed to authenticate the request through basic credentials. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The client is not authenticated. + "404": + content: + application/json: + example: + userMessage: The resource could not be found. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Resource not found. + "500": + content: + application/json: + example: + userMessage: Internal server error. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Internal server error. + summary: Get Job. + tags: + - Jobs +components: + parameters: + PageSizeQueryParam: + description: | + The maximum number of resources to retrieve. + + If not set, the default limit of `10` is be used. The maximum value for this parameter is `50`. + explode: true + in: query + name: pageSize + required: false + schema: + default: 10 + maximum: 50 + minimum: 1 + type: integer + style: form + CursorQueryParam: + description: "The cursor pointing to the first resource to be included in the\ + \ response. \nThis cursor cannot be created and must have been extracted from\ + \ a response returned by a previous API call.\n\nIf this parameter is missing,\ + \ the API returns the resources starting from the first available resource,\ + \ at index `0`.\n" + explode: true + in: query + name: cursor + required: false + schema: + type: string + style: form + UserQueryParam: + description: The ID of the user who created the searched job. + explode: true + in: query + name: user + required: false + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: form + JobIdQueryParam: + description: The ID of the job. + explode: false + in: path + name: jobId + required: true + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: simple + MessageQueryParam: + description: An optional message for the job deletion. + explode: true + in: query + name: message + required: false + schema: + default: Deleted by user + maxLength: 128 + type: string + style: form + NameQueryParam: + description: The name of the job to search for. + explode: true + in: query + name: name + required: false + schema: + example: Sample job + type: string + style: form + NameMatchModeQueryParam: + description: "The match mode used to compare name. If the match\ + \ mode is EXACT the search is case-sensitive, otherwise the search\ + \ is case-insensitive." + explode: true + in: query + name: nameMatchMode + required: false + schema: + default: ANYWHERE + enum: + - START + - END + - ANYWHERE + - EXACT + example: EXACT + type: string + style: form + ResultQueryParam: + description: The list of results which the jobs should be searched for. + explode: true + in: query + name: result + required: false + schema: + items: + enum: + - NOT_SET + - SUCCESS + - COMPLETED_WITH_ERROR + - FAILURE + - ABORTED + example: SUCCESS + type: string + type: array + style: form + SortFieldQueryParam: + description: "The field that should be used as reference for sorting, \nin addition\ + \ to the CREATED_ON field in DESC order.\n" + explode: true + in: query + name: sortField + required: false + schema: + enum: + - NAME + - TYPE + - START_DATE + - END_DATE + - STATE + - RESULT + example: NAME + type: string + style: form + SortOrderQueryParam: + description: The order of sorting. + explode: true + in: query + name: sortOrder + required: false + schema: + default: DESC + enum: + - ASC + - DESC + example: ASC + type: string + style: form + StateQueryParam: + description: The list of states which the jobs should be searched for. + explode: true + in: query + name: state + required: false + schema: + items: + enum: + - WAITING + - RUNNING + - COMPLETED + - FAILED + - DELETED + example: COMPLETED + type: string + type: array + style: form + TypeQueryParam: + description: The list of types of the jobs. + explode: true + in: query + name: type + required: false + schema: + items: + example: IMPORT + type: string + type: array + style: form + responses: + "400_Bad_Request": + content: + application/json: + example: + userMessage: The request is not valid. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The request is not valid. + "401_Unauthorized": + content: + application/json: + example: + userMessage: Failed to authenticate the request through basic credentials. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The client is not authenticated. + "500_InternalServerError": + content: + application/json: + example: + userMessage: Internal server error. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Internal server error. + "404_Not_Found": + content: + application/json: + example: + userMessage: The resource could not be found. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Resource not found. + schemas: + JobDelete: + allOf: + - $ref: "#/components/schemas/JobMessage" + deprecated: true + description: "(Deprecated) Represents a body for delete job. Use the message\ + \ query parameter instead. Deprecated, will be removed in 2026.06." + example: + message: Import finished in 12s. + JobPagedResponse: + example: + nextCursor: Y3Vyc29yV2hLblRoZXJlSXNNb3JlRGF0YQo + results: + - endDate: 2017-07-21T17:32:28Z + lastModifiedBy: a073ff90-e7bc-4b35-ba90-c4d475f642fe + message: Import finished in 12s. + type: SAMPLE + createdOn: 2017-07-21T17:32:28Z + lastModifiedOn: 2017-07-21T17:32:28Z + result: SUCCESS + createdBy: 4d250cc5-e583-4640-9874-b93d82c7a6cb + name: Sample job. + progressPercentage: 50 + id: 495756bd-419c-4fea-beb4-8a70253d8079 + state: COMPLETED + user: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + selfManaged: false + startDate: 2017-07-21T17:32:28Z + - endDate: 2017-07-21T17:32:28Z + lastModifiedBy: a073ff90-e7bc-4b35-ba90-c4d475f642fe + message: Import finished in 12s. + type: SAMPLE + createdOn: 2017-07-21T17:32:28Z + lastModifiedOn: 2017-07-21T17:32:28Z + result: SUCCESS + createdBy: 4d250cc5-e583-4640-9874-b93d82c7a6cb + name: Sample job. + progressPercentage: 50 + id: 495756bd-419c-4fea-beb4-8a70253d8079 + state: COMPLETED + user: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + selfManaged: false + startDate: 2017-07-21T17:32:28Z + properties: + nextCursor: + description: "The cursor pointing to the next page. If the cursor is missing,\ + \ there are no additional pages of resources available after the current\ + \ one." + example: Y3Vyc29yV2hLblRoZXJlSXNNb3JlRGF0YQo + type: string + results: + description: The list of results. + items: + $ref: "#/components/schemas/Job" + type: array + type: object + Job: + allOf: + - properties: + id: + description: The UUID of the represented object (entity). + example: 495756bd-419c-4fea-beb4-8a70253d8079 + format: uuid + type: string + createdBy: + description: The UUID of the user that created this resource. + example: 4d250cc5-e583-4640-9874-b93d82c7a6cb + format: uuid + type: string + createdOn: + description: Date and time the job was created on. + example: 2017-07-21T17:32:28Z + format: date-time + type: string + lastModifiedBy: + description: The UUID of the user who modified this resource the last + time. + example: a073ff90-e7bc-4b35-ba90-c4d475f642fe + format: uuid + type: string + lastModifiedOn: + description: Date and time the job was modified on. + example: 2017-07-21T17:32:28Z + format: date-time + type: string + user: + description: The UUID of the user that initiated this job. + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + selfManaged: + default: false + description: "Whether this is a self managed job, processed outside of\ + \ DG." + example: false + type: boolean + required: + - id + type: object + - $ref: "#/components/schemas/JobEndDate" + - $ref: "#/components/schemas/JobMessage" + - $ref: "#/components/schemas/JobName" + - $ref: "#/components/schemas/JobProgressPercentage" + - $ref: "#/components/schemas/JobResult" + - $ref: "#/components/schemas/JobStartDate" + - $ref: "#/components/schemas/JobState" + - $ref: "#/components/schemas/JobType" + description: Represents a job. + example: + endDate: 2017-07-21T17:32:28Z + lastModifiedBy: a073ff90-e7bc-4b35-ba90-c4d475f642fe + message: Import finished in 12s. + type: SAMPLE + createdOn: 2017-07-21T17:32:28Z + lastModifiedOn: 2017-07-21T17:32:28Z + result: SUCCESS + createdBy: 4d250cc5-e583-4640-9874-b93d82c7a6cb + name: Sample job. + progressPercentage: 50 + id: 495756bd-419c-4fea-beb4-8a70253d8079 + state: COMPLETED + user: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + selfManaged: false + startDate: 2017-07-21T17:32:28Z + JobEndDate: + properties: + endDate: + description: Date and time the job was ended on. + example: 2017-07-21T17:32:28Z + format: date-time + type: string + type: object + JobMessage: + properties: + message: + description: The message of the job. + example: Import finished in 12s. + type: string + type: object + JobName: + properties: + name: + description: The name of the job. + example: Sample job. + type: string + type: object + JobProgressPercentage: + properties: + progressPercentage: + description: The progress percentage of the job. + example: 50 + format: int32 + type: integer + type: object + JobResult: + properties: + result: + description: | + The result of the job. + + Possible values are: + * NOT_SET + * SUCCESS + * COMPLETED_WITH_ERROR + * FAILURE + * ABORTED + example: SUCCESS + type: string + type: object + JobStartDate: + properties: + startDate: + description: Date and time the job was started on. + example: 2017-07-21T17:32:28Z + format: date-time + type: string + type: object + JobState: + properties: + state: + description: | + The state of the job. + + Possible values are: + * WAITING + * RUNNING + * COMPLETED + * FAILED + * DELETED + example: COMPLETED + type: string + type: object + JobType: + properties: + type: + description: The type of the job. + example: SAMPLE + type: string + type: object + StandardErrorResponse: + example: + userMessage: The request is not valid. + titleMessage: Bad Request + errorContext: a073ff90-e7bc-4b35-ba90-c4d475f642fe + errorCode: invalidRequest + helpMessage: Check the request and try again. + statusCode: 400 + properties: + statusCode: + description: HTTP response code + example: 400 + type: integer + titleMessage: + description: The title of the error. + example: Bad Request + type: string + helpMessage: + description: A message that can help the user to understand the error. + example: Check the request and try again. + type: string + userMessage: + description: A message that can be shown to the user. + example: The request is not valid. + type: string + errorCode: + description: The error code. + example: invalidRequest + type: string + errorContext: + description: Span context ID related to the error. + example: a073ff90-e7bc-4b35-ba90-c4d475f642fe + type: string + required: + - errorCode + - statusCode + type: object + securitySchemes: + basicAuth: + description: Collibra REST API authentication using Basic Authentication. + scheme: basic + type: http + jwtAuth: + bearerFormat: JWT + description: Collibra REST API authentication using JSON Web Token. + scheme: bearer + type: http diff --git a/pkg/chip/server.go b/pkg/chip/server.go index 543b64a..6cabe71 100644 --- a/pkg/chip/server.go +++ b/pkg/chip/server.go @@ -117,7 +117,17 @@ type ServerToolConfig struct { DisabledTools []string // EnableDebugTools, when true, registers debug tools that are otherwise hidden. EnableDebugTools bool - Experimental []string + // AllowLocalFileUpload, when true, registers upload_file with its filePath + // input (reads a file directly off this process's local disk). Only meaningful + // for a chip instance running locally with access to the caller's own + // filesystem (the standalone chip binary) — a shared/remote deployment (e.g. + // chip-service) must never set this, since there "local disk" means the + // server container's disk, not any user's machine. There is deliberately no + // environment-variable escape hatch checked at call time for this — it is + // gated purely by whether the binary's own startup wiring passes true here, + // so chip-service's code has no path that could turn it on even by mistake. + AllowLocalFileUpload bool + Experimental []string // SkillsDir is the optional path to an external skills directory whose // contents are merged on top of the embedded catalog. Empty means the // embedded catalog alone is served. Only consulted when the "skills" diff --git a/pkg/clients/catalog_database_client.go b/pkg/clients/catalog_database_client.go new file mode 100644 index 0000000..6a998a5 --- /dev/null +++ b/pkg/clients/catalog_database_client.go @@ -0,0 +1,249 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +const catalogDatabaseBasePath = "/rest/catalogDatabase/v1" + +// DatabaseConnection is a discovered connection to a specific database (catalog) on a +// data source, returned by GET /databaseConnections. +type DatabaseConnection struct { + ID string `json:"id"` + Name string `json:"name"` + EdgeConnectionID string `json:"edgeConnectionId"` + DatabaseID string `json:"databaseId,omitempty"` +} + +// SchemaConnection is a discovered connection to a specific schema within a database, +// returned by GET /schemaConnections. +type SchemaConnection struct { + ID string `json:"id"` + Name string `json:"name"` + DatabaseConnectionID string `json:"databaseConnectionId"` + SchemaID string `json:"schemaId,omitempty"` +} + +// Database is a registered Database asset, returned by POST/GET /databases. +type Database struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + CommunityID string `json:"communityId"` + OwnerIDs []string `json:"ownerIds"` + ParentSystemID string `json:"parentSystemId"` + DatabaseConnectionID string `json:"databaseConnectionId"` + EdgeConnectionStatus string `json:"edgeConnectionStatus,omitempty"` +} + +// AddDatabaseRequest is the request body for POST /databases. +type AddDatabaseRequest struct { + DatabaseConnectionID string `json:"databaseConnectionId"` + CommunityID string `json:"communityId"` + ParentSystemID string `json:"parentSystemId"` + OwnerIDs []string `json:"ownerIds"` + Description string `json:"description,omitempty"` +} + +// MetadataSynchronizationRule defines which tables of a schema get synchronized and +// what additional metadata is captured for them. +type MetadataSynchronizationRule struct { + Include string `json:"include,omitempty"` + Exclude string `json:"exclude,omitempty"` + TargetDomainID string `json:"targetDomainId,omitempty"` + SkipViews bool `json:"skipViews,omitempty"` + RegisterSourceTags bool `json:"registerSourceTags,omitempty"` + IngestSemanticViews bool `json:"ingestSemanticViews,omitempty"` + RegisterDataUsageStatistics bool `json:"registerDataUsageStatistics,omitempty"` +} + +// SchemaMetadataConfiguration links a schema connection to the synchronization rules +// that decide what gets ingested for it. +type SchemaMetadataConfiguration struct { + SchemaConnectionID string `json:"schemaConnectionId"` + SynchronizationRules []MetadataSynchronizationRule `json:"synchronizationRules,omitempty"` +} + +// CatalogJob represents an asynchronous job triggered by the database (catalog) +// registration API, e.g. POST /databases/{id}/synchronizeMetadata. +type CatalogJob struct { + ID string `json:"id"` + State string `json:"state,omitempty"` + Result string `json:"result,omitempty"` + Message string `json:"message,omitempty"` +} + +// DatabaseMetadataSynchronizationRequest is the request body for +// POST /databases/{databaseId}/synchronizeMetadata. +type DatabaseMetadataSynchronizationRequest struct { + SchemaConnectionIDs []string `json:"schemaConnectionIds,omitempty"` +} + +type pagedResponse[T any] struct { + Results []T `json:"results"` +} + +// RefreshDatabaseConnections asynchronously refreshes the database connections known +// for an Edge connection via POST /databaseConnections/refresh?edgeConnectionId=... +// The refresh runs in the background; call ListDatabaseConnections afterwards (with +// retries) to see newly discovered connections. +func RefreshDatabaseConnections(ctx context.Context, client *http.Client, edgeConnectionID string) error { + endpoint := catalogDatabaseBasePath + "/databaseConnections/refresh?edgeConnectionId=" + edgeConnectionID + return doCatalogDatabasePost(ctx, client, endpoint, nil, http.StatusAccepted) +} + +// ListDatabaseConnections lists database connections discovered for an Edge connection +// via GET /databaseConnections?edgeConnectionId=... +func ListDatabaseConnections(ctx context.Context, client *http.Client, edgeConnectionID string) ([]DatabaseConnection, error) { + endpoint := catalogDatabaseBasePath + "/databaseConnections?edgeConnectionId=" + edgeConnectionID + var response pagedResponse[DatabaseConnection] + if err := doCatalogDatabaseGet(ctx, client, endpoint, &response); err != nil { + return nil, err + } + return response.Results, nil +} + +// RegisterDatabase creates a Database asset from a discovered database connection via +// POST /databases. +func RegisterDatabase(ctx context.Context, client *http.Client, request AddDatabaseRequest) (*Database, error) { + body, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("registering database: marshaling request: %w", err) + } + var database Database + if err := doCatalogDatabasePostJSON(ctx, client, catalogDatabaseBasePath+"/databases", body, http.StatusCreated, &database); err != nil { + return nil, err + } + return &database, nil +} + +// RefreshSchemaConnections asynchronously refreshes the schema connections known for a +// database connection via POST /schemaConnections/refresh?databaseConnectionId=... +func RefreshSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) error { + endpoint := catalogDatabaseBasePath + "/schemaConnections/refresh?databaseConnectionId=" + databaseConnectionID + return doCatalogDatabasePost(ctx, client, endpoint, nil, http.StatusAccepted) +} + +// ListSchemaConnections lists schema connections discovered for a database connection +// via GET /schemaConnections?databaseConnectionId=... +func ListSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) ([]SchemaConnection, error) { + endpoint := catalogDatabaseBasePath + "/schemaConnections?databaseConnectionId=" + databaseConnectionID + var response pagedResponse[SchemaConnection] + if err := doCatalogDatabaseGet(ctx, client, endpoint, &response); err != nil { + return nil, err + } + return response.Results, nil +} + +// SetSchemaMetadataConfigurationsBatch defines synchronization rules for one or more +// schema connections via POST /schemaMetadataConfigurations/batch. +func SetSchemaMetadataConfigurationsBatch(ctx context.Context, client *http.Client, configurations []SchemaMetadataConfiguration) ([]SchemaMetadataConfiguration, error) { + body, err := json.Marshal(configurations) + if err != nil { + return nil, fmt.Errorf("setting schema metadata configurations: marshaling request: %w", err) + } + var result []SchemaMetadataConfiguration + if err := doCatalogDatabasePostJSON(ctx, client, catalogDatabaseBasePath+"/schemaMetadataConfigurations/batch", body, http.StatusCreated, &result); err != nil { + return nil, err + } + return result, nil +} + +// SynchronizeDatabaseMetadata starts the jdbc-ingestion capability run for a registered +// Database asset via POST /databases/{databaseId}/synchronizeMetadata. An empty +// schemaConnectionIds list synchronizes all schemas that already have a +// SchemaMetadataConfiguration. +func SynchronizeDatabaseMetadata(ctx context.Context, client *http.Client, databaseID string, schemaConnectionIDs []string) (*CatalogJob, error) { + body, err := json.Marshal(DatabaseMetadataSynchronizationRequest{SchemaConnectionIDs: schemaConnectionIDs}) + if err != nil { + return nil, fmt.Errorf("starting ingestion: marshaling request: %w", err) + } + endpoint := catalogDatabaseBasePath + "/databases/" + databaseID + "/synchronizeMetadata" + var job CatalogJob + if err := doCatalogDatabasePostJSON(ctx, client, endpoint, body, http.StatusAccepted, &job); err != nil { + return nil, err + } + return &job, nil +} + +func doCatalogDatabaseGet(ctx context.Context, client *http.Client, endpoint string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("calling %s: building request: %w", endpoint, err) + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("calling %s: sending request: %w", endpoint, err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("calling %s: reading response: %w", endpoint, err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("calling %s: unexpected status %d: %s", endpoint, resp.StatusCode, string(body)) + } + + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("calling %s: decoding response: %w", endpoint, err) + } + + return nil +} + +func doCatalogDatabasePost(ctx context.Context, client *http.Client, endpoint string, body []byte, expectedStatus int) error { + return doCatalogDatabasePostJSON(ctx, client, endpoint, body, expectedStatus, nil) +} + +func doCatalogDatabasePostJSON(ctx context.Context, client *http.Client, endpoint string, body []byte, expectedStatus int, out any) error { + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, reader) + if err != nil { + return fmt.Errorf("calling %s: building request: %w", endpoint, err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("calling %s: sending request: %w", endpoint, err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("calling %s: reading response: %w", endpoint, err) + } + + if resp.StatusCode != expectedStatus { + switch resp.StatusCode { + case http.StatusConflict: + return fmt.Errorf("calling %s: conflict, an operation may already be in progress: %s", endpoint, string(respBody)) + default: + return fmt.Errorf("calling %s: unexpected status %d: %s", endpoint, resp.StatusCode, string(respBody)) + } + } + + if out == nil || len(respBody) == 0 { + return nil + } + + if err := json.Unmarshal(respBody, out); err != nil { + return fmt.Errorf("calling %s: decoding response: %w", endpoint, err) + } + + return nil +} diff --git a/pkg/clients/data_source_docs_client.go b/pkg/clients/data_source_docs_client.go new file mode 100644 index 0000000..587e373 --- /dev/null +++ b/pkg/clients/data_source_docs_client.go @@ -0,0 +1,356 @@ +// Package clients: data_source_docs_client.go fetches connection setup guidance +// (driver class, JDBC connection string format, and per-auth-method connection +// properties) for any of Collibra's ~68 supported data sources, from Collibra's +// public product documentation site. +// +// IMPORTANT — this is a documented gap, not a stable integration: +// +// There is no published/versioned API for this. The URLs and HTML structure below +// were reverse-engineered by reading the JavaScript +// (productresources.collibra.com/docs/catalog-connectors/Content/Resources/JavaScripts/power-catcon.js) +// that Collibra's own docs site uses internally to render its "data source" selector +// widget. Collibra could restructure or remove these pages at any time without +// notice, since they were never intended to be consumed as an API. Treat every +// function here as best-effort: a fetch or parse failure means "fall back to asking +// the user for the connection properties directly," not a hard error condition to +// retry aggressively. +// +// If this breaks: re-derive the URL pattern by fetching +// https://productresources.collibra.com/docs/collibra/latest/Content/Edge/JDBCConnections/ta_create-jdbc-connection.htm?data-source= +// in a real browser with devtools open, and inspecting network requests fired by +// power-catcon.js's fetchCatConDocs() function. +package clients + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + "golang.org/x/net/html" +) + +const catalogConnectorsBaseURL = "https://productresources.collibra.com/docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors" + +// DataSourceSlug identifies one of Collibra's supported data sources in the docs +// site's connector selector (e.g. {ID: "snowflake", Name: "Snowflake"}). +type DataSourceSlug struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// DataSourceInfo is driver/connection-string level documentation for a data source. +type DataSourceInfo struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + DriverClassName string `json:"driverClassName,omitempty"` + ConnectionStringFormat string `json:"connectionStringFormat,omitempty"` + MarketplaceURL string `json:"marketplaceUrl,omitempty"` + Prerequisites string `json:"prerequisites,omitempty"` +} + +// ConnectionPropertyDoc is one documented connection property row. +type ConnectionPropertyDoc struct { + Required bool `json:"required"` + Name string `json:"name"` + Type string `json:"type"` + ValueType string `json:"valueType"` + Description string `json:"description,omitempty"` +} + +// AuthMethodProperties groups the connection properties documented for one +// authentication method (e.g. "Key pair (unencrypted)", "OAuth"). +type AuthMethodProperties struct { + Name string `json:"name"` + Properties []ConnectionPropertyDoc `json:"properties"` +} + +// ListDataSources fetches the full list of data sources shown in Collibra's docs +// connector selector (68 as of 2026-07-04). +func ListDataSources(ctx context.Context, client *http.Client) ([]DataSourceSlug, error) { + doc, err := fetchAndParseHTML(ctx, client, catalogConnectorsBaseURL+"/_all-data-sources/selector/selector.htm") + if err != nil { + return nil, fmt.Errorf("listing data sources: %w", err) + } + + var sources []DataSourceSlug + seen := map[string]bool{} + for _, div := range htmlFindAll(doc, isElementWithClass("div", "catalogConnector")) { + id := htmlAttr(div, "id") + if id == "" || seen[id] { + continue + } + seen[id] = true + sources = append(sources, DataSourceSlug{ID: id, Name: strings.TrimSpace(htmlText(div))}) + } + + if len(sources) == 0 { + return nil, fmt.Errorf("listing data sources: no connectors found — the docs site's page structure may have changed") + } + return sources, nil +} + +// GetDataSourceInfo fetches driver class name, JDBC connection string format, +// Marketplace listing URL, and prerequisites for a data source, by its slug (as +// returned by ListDataSources). +func GetDataSourceInfo(ctx context.Context, client *http.Client, slug string) (*DataSourceInfo, error) { + doc, err := fetchAndParseHTML(ctx, client, catalogConnectorsBaseURL+"/"+slug+"/data-source-information.htm") + if err != nil { + return nil, fmt.Errorf("getting data source info for %q: %w", slug, err) + } + + info := &DataSourceInfo{ID: slug} + + if el := htmlFindByID(doc, "catcon-driver"); el != nil { + info.DriverClassName = strings.TrimSpace(htmlText(el)) + } + if el := htmlFindByID(doc, "catcon-jdbc-url"); el != nil { + if code := htmlFind(el, isElement("code")); code != nil { + info.ConnectionStringFormat = strings.TrimSpace(htmlText(code)) + } + } + if el := htmlFindByID(doc, "catcon-marketplace"); el != nil { + if a := htmlFind(el, isElement("a")); a != nil { + info.MarketplaceURL = htmlAttr(a, "href") + } + } + if el := htmlFindByID(doc, "catcon-prerequisites"); el != nil { + info.Prerequisites = collapseWhitespace(htmlText(el)) + } + + if info.DriverClassName == "" && info.ConnectionStringFormat == "" { + return nil, fmt.Errorf("getting data source info for %q: no recognizable content found — the docs site's page structure may have changed, or %q is not a valid slug", slug, slug) + } + + return info, nil +} + +// GetConnectionProperties fetches the documented connection properties for a data +// source, grouped by authentication method (e.g. a Snowflake connection documents +// separate property sets for "Username and password", "Key pair (unencrypted)", +// "OAuth", etc.). Use this to know exactly which named properties to pass as +// edge_create_connection's additionalProperties for a given auth method. +func GetConnectionProperties(ctx context.Context, client *http.Client, slug string) ([]AuthMethodProperties, error) { + doc, err := fetchAndParseHTML(ctx, client, catalogConnectorsBaseURL+"/"+slug+"/minimum-connection-properties-edge.htm") + if err != nil { + return nil, fmt.Errorf("getting connection properties for %q: %w", slug, err) + } + + const subselectorPrefix = "data-sources/catalog-connector-subselectors." + + // Map auth-method key (e.g. "key-pair-unencrypted") to its human-readable name, + // read from the radio button selector's and + // sibling NAME. + names := map[string]string{} + for _, input := range htmlFindAll(doc, isElement("input")) { + value := htmlAttr(input, "value") + if !strings.HasPrefix(value, subselectorPrefix) { + continue + } + key := strings.TrimPrefix(value, subselectorPrefix) + if label := input.Parent; label != nil { + if span := htmlFind(label, isElement("span")); span != nil { + names[key] = strings.TrimSpace(htmlText(span)) + } + } + } + + var methods []AuthMethodProperties + seen := map[string]bool{} + for _, div := range htmlFindAll(doc, isElement("div")) { + cond := htmlAttr(div, "data-mc-conditions") + if cond == "" { + continue + } + + var key string + for _, tok := range strings.Split(cond, ",") { + tok = strings.TrimSpace(tok) + if k, ok := strings.CutPrefix(tok, subselectorPrefix); ok && k != "all" { + key = k + break + } + } + if key == "" || seen[key] { + continue + } + + table := htmlFind(div, isElement("table")) + if table == nil { + continue + } + thead := htmlFind(table, isElement("thead")) + if thead == nil || !strings.Contains(strings.ToLower(htmlText(thead)), "required") { + // Not a connection-properties table (this div's condition may gate some + // other kind of content) — skip rather than misparse it. + continue + } + + seen[key] = true + name := names[key] + if name == "" { + name = key + } + methods = append(methods, AuthMethodProperties{Name: name, Properties: parsePropertyRows(table)}) + } + + if len(methods) == 0 { + return nil, fmt.Errorf("getting connection properties for %q: no auth-method property tables found — the docs site's page structure may have changed, or this data source may not document any (some sources need no extra properties beyond the driver/connection string)", slug) + } + + return methods, nil +} + +func parsePropertyRows(table *html.Node) []ConnectionPropertyDoc { + tbody := htmlFind(table, isElement("tbody")) + if tbody == nil { + return nil + } + + var props []ConnectionPropertyDoc + for _, tr := range directChildren(tbody, "tr") { + tds := directChildren(tr, "td") + if len(tds) < 5 { + continue + } + + required := false + if img := htmlFind(tds[0], isElement("img")); img != nil { + required = strings.EqualFold(htmlAttr(img, "alt"), "yes") + } else { + required = strings.EqualFold(strings.TrimSpace(htmlText(tds[0])), "yes") + } + + props = append(props, ConnectionPropertyDoc{ + Required: required, + Name: strings.TrimSpace(htmlText(tds[1])), + Type: strings.TrimSpace(htmlText(tds[2])), + ValueType: strings.TrimSpace(htmlText(tds[3])), + Description: collapseWhitespace(htmlText(tds[4])), + }) + } + return props +} + +func fetchAndParseHTML(ctx context.Context, client *http.Client, url string) (*html.Node, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("building request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("not found (%s)", url) + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + doc, err := html.Parse(resp.Body) + if err != nil { + return nil, fmt.Errorf("parsing HTML: %w", err) + } + return doc, nil +} + +// --- Minimal HTML tree helpers (avoids pulling in a full CSS-selector library for +// what is otherwise a handful of narrow, structure-specific lookups). --- + +func htmlFindAll(n *html.Node, match func(*html.Node) bool) []*html.Node { + var out []*html.Node + var walk func(*html.Node) + walk = func(n *html.Node) { + if match(n) { + out = append(out, n) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return out +} + +func htmlFind(n *html.Node, match func(*html.Node) bool) *html.Node { + if match(n) { + return n + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if found := htmlFind(c, match); found != nil { + return found + } + } + return nil +} + +func htmlFindByID(n *html.Node, id string) *html.Node { + return htmlFind(n, func(n *html.Node) bool { + return n.Type == html.ElementNode && htmlAttr(n, "id") == id + }) +} + +func isElement(tag string) func(*html.Node) bool { + return func(n *html.Node) bool { return n.Type == html.ElementNode && n.Data == tag } +} + +func isElementWithClass(tag, class string) func(*html.Node) bool { + return func(n *html.Node) bool { + if n.Type != html.ElementNode || n.Data != tag { + return false + } + for _, c := range strings.Fields(htmlAttr(n, "class")) { + if c == class { + return true + } + } + return false + } +} + +// directChildren returns n's immediate element children matching tag (not a +// recursive search) — used for table row/cell parsing where a cell's own content +// (e.g. a collapsible "steps" dropdown) could otherwise be mistaken for sibling rows. +func directChildren(n *html.Node, tag string) []*html.Node { + var out []*html.Node + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.Data == tag { + out = append(out, c) + } + } + return out +} + +func htmlAttr(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +func htmlText(n *html.Node) string { + var sb strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.TextNode { + sb.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return sb.String() +} + +func collapseWhitespace(s string) string { + return strings.Join(strings.Fields(s), " ") +} diff --git a/pkg/clients/data_source_docs_client_test.go b/pkg/clients/data_source_docs_client_test.go new file mode 100644 index 0000000..a19f048 --- /dev/null +++ b/pkg/clients/data_source_docs_client_test.go @@ -0,0 +1,150 @@ +package clients_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" +) + +const selectorFixture = ` +
    +
  • Snowflake
  • +
  • PostgreSQL
  • +
  • +
+` + +const dataSourceInfoFixture = ` +
+

The JDBC connection string:

+

jdbc:snowflake://<accountname>.snowflakecomputing.com

+

The JDBC driver class name:

+
net.snowflake.client.jdbc.SnowflakeDriver
+

The Collibra Marketplace listing URL:

+ +

The connection prerequisites:

+
  • Download the driver from Marketplace.
  • +
    +` + +const connectionPropertiesFixture = ` +
    + + +
    +
    +

    To establish a connection using username and password authentication.

    + + + + +

    Required?

    Name

    Type

    Value Type

    Value

    YesPasswordTextSecretThe password.
    NoroleTextPlaintextThe role.
    +
    +
    + + + +

    Required?

    Name

    Type

    Value Type

    Value

    Yesprivate_key_fileFilePlaintextThe key file.
    +
    +` + +func newDocsTestServer(t *testing.T) *http.Client { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/_all-data-sources/selector/selector.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(selectorFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/snowflake/data-source-information.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(dataSourceInfoFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/snowflake/minimum-connection-properties-edge.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(connectionPropertiesFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/does-not-exist/data-source-information.htm", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + return testutil.NewClient(httptest.NewServer(mux)) +} + +func TestListDataSources(t *testing.T) { + client := newDocsTestServer(t) + sources, err := clients.ListDataSources(t.Context(), client) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(sources) != 2 { + t.Fatalf("expected 2 data sources (hidden placeholder excluded), got %d: %+v", len(sources), sources) + } + if sources[0].ID != "snowflake" || sources[0].Name != "Snowflake" { + t.Fatalf("unexpected first source: %+v", sources[0]) + } +} + +func TestGetDataSourceInfo(t *testing.T) { + client := newDocsTestServer(t) + info, err := clients.GetDataSourceInfo(t.Context(), client, "snowflake") + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if info.DriverClassName != "net.snowflake.client.jdbc.SnowflakeDriver" { + t.Fatalf("unexpected driver class: %s", info.DriverClassName) + } + if info.ConnectionStringFormat != "jdbc:snowflake://.snowflakecomputing.com" { + t.Fatalf("unexpected connection string: %s", info.ConnectionStringFormat) + } + if info.MarketplaceURL != "https://marketplace.collibra.com/listings/snowflake" { + t.Fatalf("unexpected marketplace url: %s", info.MarketplaceURL) + } + if info.Prerequisites == "" { + t.Fatalf("expected prerequisites to be populated") + } +} + +func TestGetDataSourceInfo_NotFound(t *testing.T) { + client := newDocsTestServer(t) + _, err := clients.GetDataSourceInfo(t.Context(), client, "does-not-exist") + if err == nil { + t.Fatalf("expected an error for a 404") + } +} + +func TestGetConnectionProperties(t *testing.T) { + client := newDocsTestServer(t) + methods, err := clients.GetConnectionProperties(t.Context(), client, "snowflake") + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(methods) != 2 { + t.Fatalf("expected 2 auth methods, got %d: %+v", len(methods), methods) + } + + byName := map[string]clients.AuthMethodProperties{} + for _, m := range methods { + byName[m.Name] = m + } + + userPass, ok := byName["Username and password"] + if !ok { + t.Fatalf("expected 'Username and password' auth method, got: %+v", methods) + } + if len(userPass.Properties) != 2 { + t.Fatalf("expected 2 properties, got: %+v", userPass.Properties) + } + if !userPass.Properties[0].Required || userPass.Properties[0].Name != "Password" || userPass.Properties[0].ValueType != "Secret" { + t.Fatalf("unexpected Password property: %+v", userPass.Properties[0]) + } + if userPass.Properties[1].Required { + t.Fatalf("expected role to be not required: %+v", userPass.Properties[1]) + } + + keyPair, ok := byName["Key pair (encrypted)"] + if !ok { + t.Fatalf("expected 'Key pair (encrypted)' auth method, got: %+v", methods) + } + if len(keyPair.Properties) != 1 || keyPair.Properties[0].Name != "private_key_file" || keyPair.Properties[0].Type != "File" { + t.Fatalf("unexpected key pair properties: %+v", keyPair.Properties) + } +} diff --git a/pkg/clients/dgc_community_client.go b/pkg/clients/dgc_community_client.go new file mode 100644 index 0000000..472e521 --- /dev/null +++ b/pkg/clients/dgc_community_client.go @@ -0,0 +1,96 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// CommunityDetails is the response shape returned by POST /rest/2.0/communities. +type CommunityDetails struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` +} + +// CreateCommunityRequest is the request body for POST /rest/2.0/communities. +type CreateCommunityRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` +} + +// CreateCommunity creates a new community via POST /rest/2.0/communities. +func CreateCommunity(ctx context.Context, client *http.Client, request CreateCommunityRequest) (*CommunityDetails, error) { + var community CommunityDetails + if err := postJSON(ctx, client, "/rest/2.0/communities", request, http.StatusCreated, &community); err != nil { + return nil, fmt.Errorf("creating community: %w", err) + } + return &community, nil +} + +// DomainDetails is the response shape returned by POST /rest/2.0/domains. +type DomainDetails struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + CommunityID string `json:"communityId"` + TypeID string `json:"typeId"` +} + +// CreateDomainRequest is the request body for POST /rest/2.0/domains. +type CreateDomainRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + CommunityID string `json:"communityId"` + TypeID string `json:"typeId"` + ExcludedFromAutoHyperlinking bool `json:"excludedFromAutoHyperlinking,omitempty"` +} + +// CreateDomain creates a new domain via POST /rest/2.0/domains. +func CreateDomain(ctx context.Context, client *http.Client, request CreateDomainRequest) (*DomainDetails, error) { + var domain DomainDetails + if err := postJSON(ctx, client, "/rest/2.0/domains", request, http.StatusCreated, &domain); err != nil { + return nil, fmt.Errorf("creating domain: %w", err) + } + return &domain, nil +} + +// postJSON is a small shared helper for simple "POST body, decode JSON response" +// calls against DGC's rest/2.0 API family. +func postJSON(ctx context.Context, client *http.Client, endpoint string, request any, expectedStatus int, out any) error { + body, err := json.Marshal(request) + if err != nil { + return fmt.Errorf("marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + + if resp.StatusCode != expectedStatus { + return fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + + if err := json.Unmarshal(respBody, out); err != nil { + return fmt.Errorf("decoding response: %w", err) + } + + return nil +} diff --git a/pkg/clients/dgc_domain_type_client.go b/pkg/clients/dgc_domain_type_client.go new file mode 100644 index 0000000..e0a234b --- /dev/null +++ b/pkg/clients/dgc_domain_type_client.go @@ -0,0 +1,57 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// DomainType is a DGC domain type, as returned by GET /rest/2.0/domainTypes. +type DomainTypeDetails struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + PublicID string `json:"publicId,omitempty"` +} + +// DomainTypePagedResponse is the response shape for GET /rest/2.0/domainTypes. +type DomainTypePagedResponse struct { + Total int64 `json:"total"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` + Results []DomainTypeDetails `json:"results"` +} + +// FindDomainTypesQueryParams are the supported query parameters for +// GET /rest/2.0/domainTypes. Name matches case-insensitively as a substring by +// default (nameMatchMode=ANYWHERE server-side). +type FindDomainTypesQueryParams struct { + Name string `url:"name,omitempty"` + Limit int `url:"limit,omitempty"` + Offset int `url:"offset,omitempty"` +} + +// FindDomainTypes searches DGC domain types via GET /rest/2.0/domainTypes. +func FindDomainTypes(ctx context.Context, client *http.Client, params FindDomainTypesQueryParams) (*DomainTypePagedResponse, error) { + endpoint, err := buildUrl("/rest/2.0/domainTypes", params) + if err != nil { + return nil, fmt.Errorf("finding domain types: building endpoint: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("finding domain types: building request: %w", err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("finding domain types: %w", err) + } + + var result DomainTypePagedResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("finding domain types: decoding response: %w", err) + } + return &result, nil +} diff --git a/pkg/clients/dgc_domain_type_client_test.go b/pkg/clients/dgc_domain_type_client_test.go new file mode 100644 index 0000000..9345b2c --- /dev/null +++ b/pkg/clients/dgc_domain_type_client_test.go @@ -0,0 +1,39 @@ +package clients_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindDomainTypes(t *testing.T) { + typeID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/domainTypes", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("name") != "Physical Data Dictionary" { + t.Fatalf("unexpected name query param: %s", r.URL.Query().Get("name")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":1,"offset":0,"limit":50,"results":[{"id":"` + typeID.String() + `","name":"Physical Data Dictionary","publicId":"PhysicalDataDictionary"}]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + response, err := clients.FindDomainTypes(t.Context(), client, clients.FindDomainTypesQueryParams{Name: "Physical Data Dictionary", Limit: 50}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if response.Total != 1 || len(response.Results) != 1 { + t.Fatalf("unexpected response: %+v", response) + } + if response.Results[0].ID != typeID.String() { + t.Fatalf("expected id %s, got %s", typeID.String(), response.Results[0].ID) + } +} diff --git a/pkg/clients/dgc_job_client.go b/pkg/clients/dgc_job_client.go new file mode 100644 index 0000000..bbe7e11 --- /dev/null +++ b/pkg/clients/dgc_job_client.go @@ -0,0 +1,123 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// Job is a DGC job as returned by the Jobs API — GET /rest/jobs/v1/jobs and +// GET /rest/jobs/v1/jobs/{jobId}. Its fields mirror the Job schema in job-api.yaml. +// This is the DGC-wide job resource you poll for status (e.g. after triggering a +// catalog database sync, whose CatalogJob response carries the id to poll here). It is +// distinct from edge-management's per-Edge-site jobs (see GetJobStatusLog / +// edge_get_job_status), which track Edge capability runs and connection tests instead. +type Job struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + State string `json:"state,omitempty"` + Result string `json:"result,omitempty"` + Message string `json:"message,omitempty"` + ProgressPercentage int `json:"progressPercentage,omitempty"` + StartDate string `json:"startDate,omitempty"` + EndDate string `json:"endDate,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + CreatedOn string `json:"createdOn,omitempty"` + LastModifiedBy string `json:"lastModifiedBy,omitempty"` + LastModifiedOn string `json:"lastModifiedOn,omitempty"` + User string `json:"user,omitempty"` + SelfManaged bool `json:"selfManaged,omitempty"` +} + +// JobPagedResponse is the paged response returned by GET /rest/jobs/v1/jobs. +type JobPagedResponse struct { + Results []Job `json:"results"` + NextCursor string `json:"nextCursor,omitempty"` +} + +type jobFindParams struct { + Name string `url:"name,omitempty"` + NameMatchMode string `url:"nameMatchMode,omitempty"` + Result []string `url:"result,omitempty"` + State []string `url:"state,omitempty"` + Type []string `url:"type,omitempty"` + User string `url:"user,omitempty"` + SortField string `url:"sortField,omitempty"` + SortOrder string `url:"sortOrder,omitempty"` + Cursor string `url:"cursor,omitempty"` + PageSize int `url:"pageSize,omitempty"` +} + +// GetJob fetches a DGC job's status via GET /rest/jobs/v1/jobs/{jobId}. This is the +// current, non-deprecated DGC job resource (the older /rest/2.0/jobs/{jobId} is +// deprecated for removal, per its own javadoc, in favor of this one). +func GetJob(ctx context.Context, client *http.Client, jobID string) (*Job, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/rest/jobs/v1/jobs/"+jobID, nil) + if err != nil { + return nil, fmt.Errorf("getting job %q: building request: %w", jobID, err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("getting job %q: %w", jobID, err) + } + + var result Job + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("getting job %q: decoding response: %w", jobID, err) + } + return &result, nil +} + +// FindJobs searches DGC jobs via GET /rest/jobs/v1/jobs, applying the given filters +// and pagination. All filters are optional. +func FindJobs( + ctx context.Context, + client *http.Client, + name string, + nameMatchMode string, + result []string, + state []string, + jobType []string, + user string, + sortField string, + sortOrder string, + cursor string, + pageSize int, +) (*JobPagedResponse, error) { + params := jobFindParams{ + Name: name, + NameMatchMode: nameMatchMode, + Result: result, + State: state, + Type: jobType, + User: user, + SortField: sortField, + SortOrder: sortOrder, + Cursor: cursor, + PageSize: pageSize, + } + + endpoint, err := buildUrl("/rest/jobs/v1/jobs", params) + if err != nil { + return nil, fmt.Errorf("finding jobs: building endpoint: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("finding jobs: building request: %w", err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("finding jobs: %w", err) + } + + var resp JobPagedResponse + if err := json.Unmarshal(body, &resp); err != nil { + return nil, fmt.Errorf("finding jobs: decoding response: %w", err) + } + return &resp, nil +} diff --git a/pkg/clients/dgc_job_client_test.go b/pkg/clients/dgc_job_client_test.go new file mode 100644 index 0000000..ed70db6 --- /dev/null +++ b/pkg/clients/dgc_job_client_test.go @@ -0,0 +1,33 @@ +package clients_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestGetJob(t *testing.T) { + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/jobs/v1/jobs/"+jobID.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"` + jobID.String() + `","name":"Database Synchronisation of source","type":"DELTA_INGESTION","state":"RUNNING","result":"NOT_SET","progressPercentage":42}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + job, err := clients.GetJob(t.Context(), client, jobID.String()) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if job.State != "RUNNING" || job.Type != "DELTA_INGESTION" || job.ProgressPercentage != 42 { + t.Fatalf("unexpected job: %+v", job) + } +} diff --git a/pkg/clients/dgc_user_client.go b/pkg/clients/dgc_user_client.go new file mode 100644 index 0000000..da3f41c --- /dev/null +++ b/pkg/clients/dgc_user_client.go @@ -0,0 +1,82 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// User is a DGC user, as returned by GET /rest/2.0/users and +// GET /rest/2.0/users/{id}. +type User struct { + ID string `json:"id"` + UserName string `json:"userName,omitempty"` + FirstName string `json:"firstName,omitempty"` + LastName string `json:"lastName,omitempty"` + EmailAddress string `json:"emailAddress,omitempty"` + Title string `json:"title,omitempty"` + Department string `json:"department,omitempty"` + Enabled bool `json:"enabled"` +} + +// UserPagedResponse is the response shape for GET /rest/2.0/users. +type UserPagedResponse struct { + Total int64 `json:"total"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` + Results []User `json:"results"` +} + +// FindUsersQueryParams are the supported query parameters for GET /rest/2.0/users. +// Name matches case-insensitively as a substring, by default across username, +// first name, last name, and first+last/last+first combinations. +type FindUsersQueryParams struct { + Name string `url:"name,omitempty"` + Limit int `url:"limit,omitempty"` + Offset int `url:"offset,omitempty"` + IncludeDisabled bool `url:"includeDisabled,omitempty"` +} + +// FindUsers searches DGC users via GET /rest/2.0/users. +func FindUsers(ctx context.Context, client *http.Client, params FindUsersQueryParams) (*UserPagedResponse, error) { + endpoint, err := buildUrl("/rest/2.0/users", params) + if err != nil { + return nil, fmt.Errorf("finding users: building endpoint: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("finding users: building request: %w", err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("finding users: %w", err) + } + + var result UserPagedResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("finding users: decoding response: %w", err) + } + return &result, nil +} + +// GetUser fetches a single DGC user by id via GET /rest/2.0/users/{id}. +func GetUser(ctx context.Context, client *http.Client, userID string) (*User, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/rest/2.0/users/"+userID, nil) + if err != nil { + return nil, fmt.Errorf("getting user %q: building request: %w", userID, err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("getting user %q: %w", userID, err) + } + + var result User + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("getting user %q: decoding response: %w", userID, err) + } + return &result, nil +} diff --git a/pkg/clients/dgc_user_client_test.go b/pkg/clients/dgc_user_client_test.go new file mode 100644 index 0000000..29f7cbc --- /dev/null +++ b/pkg/clients/dgc_user_client_test.go @@ -0,0 +1,61 @@ +package clients_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindUsers(t *testing.T) { + adminID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/users", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("name") != "Admin" { + t.Fatalf("unexpected name query param: %s", r.URL.Query().Get("name")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":1,"offset":0,"limit":20,"results":[{"id":"` + adminID.String() + `","userName":"Admin","emailAddress":"admin@example.com","enabled":true}]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + response, err := clients.FindUsers(t.Context(), client, clients.FindUsersQueryParams{Name: "Admin", Limit: 20}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if response.Total != 1 || len(response.Results) != 1 { + t.Fatalf("unexpected response: %+v", response) + } + if response.Results[0].ID != adminID.String() { + t.Fatalf("expected id %s, got %s", adminID.String(), response.Results[0].ID) + } +} + +func TestGetUser(t *testing.T) { + userID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/users/"+userID.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"` + userID.String() + `","userName":"jdoe","firstName":"Jane","lastName":"Doe","enabled":true}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + user, err := clients.GetUser(t.Context(), client, userID.String()) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if user.UserName != "jdoe" || user.FirstName != "Jane" { + t.Fatalf("unexpected user: %+v", user) + } +} diff --git a/pkg/clients/edge_capability_client.go b/pkg/clients/edge_capability_client.go new file mode 100644 index 0000000..a3c5a80 --- /dev/null +++ b/pkg/clients/edge_capability_client.go @@ -0,0 +1,194 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// CapabilityRequest is the request body for creating or updating a capability via +// POST /edge/api/rest/v2/capabilities or PUT /edge/api/rest/v2/capabilities/{id}. +type CapabilityRequest struct { + TypeID string `json:"typeId"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + EdgeSiteID string `json:"edgeSiteId"` + Parameters map[string]any `json:"parameters"` +} + +// CreateCapability creates a new Edge capability via POST /edge/api/rest/v2/capabilities. +// The server assigns the capability id. +func CreateCapability(ctx context.Context, client *http.Client, request CapabilityRequest) (*EdgeCapability, error) { + return doCapabilityRequest(ctx, client, http.MethodPost, "/edge/api/rest/v2/capabilities", request) +} + +// CreateOrUpdateCapability creates or updates an Edge capability with a known id via +// PUT /edge/api/rest/v2/capabilities/{id}. +func CreateOrUpdateCapability(ctx context.Context, client *http.Client, capabilityID string, request CapabilityRequest) (*EdgeCapability, error) { + endpoint := "/edge/api/rest/v2/capabilities/" + capabilityID + return doCapabilityRequest(ctx, client, http.MethodPut, endpoint, request) +} + +func doCapabilityRequest(ctx context.Context, client *http.Client, method, endpoint string, request CapabilityRequest) (*EdgeCapability, error) { + body, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("saving capability: marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("saving capability: building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("saving capability: sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("saving capability: reading response: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + switch resp.StatusCode { + case http.StatusBadRequest: + return nil, fmt.Errorf("saving capability: bad request (invalid parameters or typeId): %s", string(respBody)) + case http.StatusNotFound: + return nil, fmt.Errorf("saving capability: edge site not found: %s", string(respBody)) + default: + return nil, fmt.Errorf("saving capability: unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + } + + var result EdgeCapability + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("saving capability: decoding response: %w", err) + } + + return &result, nil +} + +// EdgeCapability is the full capability resource returned by the list/get/find +// endpoints. It is distinct from Capability (the create/update request-response +// shape) because these read endpoints expose the capability type as a structured +// object whose id is needed to filter integrations by platform. +type EdgeCapability struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + EdgeSiteId string `json:"edgeSiteId,omitempty"` + EdgeSiteName string `json:"edgeSiteName,omitempty"` + Type *EdgeCapabilityType `json:"type,omitempty"` + Parameters map[string]any `json:"parameters,omitempty"` +} + +type EdgeCapabilityType struct { + Id string `json:"id,omitempty"` +} + +type CapabilityFindRequest struct { + EdgeSiteId string `json:"edgeSiteId,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` +} + +type CapabilityRunRequest struct { + JobId string `json:"jobId,omitempty"` + InFastNamespace bool `json:"inFastNamespace,omitempty"` + Parameters map[string]any `json:"parameters,omitempty"` + WorkflowName string `json:"workflowName,omitempty"` +} + +// ListCapabilities lists all Edge capabilities via GET /edge/api/rest/v2/capabilities. +func ListCapabilities(ctx context.Context, client *http.Client) ([]EdgeCapability, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "/edge/api/rest/v2/capabilities", nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var capabilities []EdgeCapability + if err := json.Unmarshal(body, &capabilities); err != nil { + return nil, fmt.Errorf("failed to parse capabilities response: %w", err) + } + return capabilities, nil +} + +// FindCapabilities searches Edge capabilities via POST /edge/api/rest/v2/capabilities/find. +func FindCapabilities(ctx context.Context, client *http.Client, reqBody CapabilityFindRequest) ([]EdgeCapability, error) { + data, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "POST", "/edge/api/rest/v2/capabilities/find", bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var caps []EdgeCapability + if err := json.Unmarshal(body, &caps); err != nil { + return nil, fmt.Errorf("failed to parse capabilities response: %w", err) + } + return caps, nil +} + +// GetCapability fetches a single Edge capability via GET /edge/api/rest/v2/capabilities/{id}. +func GetCapability(ctx context.Context, client *http.Client, id string) (*EdgeCapability, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "/edge/api/rest/v2/capabilities/"+id, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var cap EdgeCapability + if err := json.Unmarshal(body, &cap); err != nil { + return nil, fmt.Errorf("failed to parse capability response: %w", err) + } + return &cap, nil +} + +// DeleteCapability deletes an Edge capability via DELETE /edge/api/rest/v2/capabilities/{id}. +func DeleteCapability(ctx context.Context, client *http.Client, id string) error { + req, err := http.NewRequestWithContext(ctx, "DELETE", "/edge/api/rest/v2/capabilities/"+id, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +// RunCapability triggers an Edge capability run via POST /edge/api/rest/v2/capabilities/{id}/run +// and returns the resulting job id. +func RunCapability(ctx context.Context, client *http.Client, id string, reqBody CapabilityRunRequest) (string, error) { + data, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "POST", "/edge/api/rest/v2/capabilities/"+id+"/run", bytes.NewBuffer(data)) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return "", err + } + var jobId string + if err := json.Unmarshal(body, &jobId); err != nil { + return "", fmt.Errorf("failed to parse run response: %w", err) + } + return jobId, nil +} diff --git a/pkg/clients/edge_connection_client.go b/pkg/clients/edge_connection_client.go new file mode 100644 index 0000000..421ce55 --- /dev/null +++ b/pkg/clients/edge_connection_client.go @@ -0,0 +1,241 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" +) + +// ConnectionRequest is the request body for creating or updating a connection via +// POST /edge/api/rest/v2/connections or PUT /edge/api/rest/v2/connections/{id}. +type ConnectionRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + TypeID string `json:"typeId"` + EdgeSiteID string `json:"edgeSiteId"` + VaultID string `json:"vaultId,omitempty"` + Parameters map[string]any `json:"parameters"` +} + +// CreateConnection creates a new Edge connection via POST /edge/api/rest/v2/connections. +// The server assigns the connection id. +func CreateConnection(ctx context.Context, client *http.Client, request ConnectionRequest) (*EdgeConnection, error) { + return doConnectionRequest(ctx, client, http.MethodPost, "/edge/api/rest/v2/connections", request) +} + +// CreateOrUpdateConnection creates or updates an Edge connection with a known id via +// PUT /edge/api/rest/v2/connections/{connectionId}. +func CreateOrUpdateConnection(ctx context.Context, client *http.Client, connectionID string, request ConnectionRequest) (*EdgeConnection, error) { + endpoint := "/edge/api/rest/v2/connections/" + connectionID + return doConnectionRequest(ctx, client, http.MethodPut, endpoint, request) +} + +// ConnectionFindRequest is the request body for POST /edge/api/rest/v2/connections/find. +type ConnectionFindRequest struct { + EdgeSiteID string `json:"edgeSiteId,omitempty"` + Name string `json:"name,omitempty"` + NameMatchMode string `json:"nameMatchMode,omitempty"` +} + +// FindConnections searches Edge connections via POST /edge/api/rest/v2/connections/find. +// Useful for picking up a connection a user created manually (e.g. via the DGC/Edge UI, +// for a driver file too large to pass through this tool) by name, instead of needing +// its id. +func FindConnections(ctx context.Context, client *http.Client, request ConnectionFindRequest) ([]EdgeConnection, error) { + body, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("finding connections: marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/edge/api/rest/v2/connections/find", bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("finding connections: building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("finding connections: sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("finding connections: reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("finding connections: unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + + var result []EdgeConnection + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("finding connections: decoding response: %w", err) + } + return result, nil +} + +func doConnectionRequest(ctx context.Context, client *http.Client, method, endpoint string, request ConnectionRequest) (*EdgeConnection, error) { + body, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("saving connection: marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("saving connection: building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("saving connection: sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("saving connection: reading response: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + switch resp.StatusCode { + case http.StatusBadRequest: + return nil, fmt.Errorf("saving connection: bad request (invalid parameters or typeId): %s", string(respBody)) + case http.StatusNotFound: + return nil, fmt.Errorf("saving connection: edge site not found: %s", string(respBody)) + default: + return nil, fmt.Errorf("saving connection: unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + } + + var result EdgeConnection + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("saving connection: decoding response: %w", err) + } + + return &result, nil +} + +// UploadFile uploads an arbitrary file via POST /edge/api/rest/v2/upload and returns +// the artifact URI (e.g. "jar:///") to reference from any FILE-type +// connection or capability parameter — confirmed (via ConnectionPropertyDeserializer) +// that the parameter value is the bare artifact URI string, not a wrapped object; this +// covers JDBC driver jars as well as certs, keytabs, private keys, etc. +func UploadFile(ctx context.Context, client *http.Client, filename string, content []byte) (string, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, err := writer.CreateFormFile("file", filename) + if err != nil { + return "", fmt.Errorf("uploading file: creating form file: %w", err) + } + if _, err := part.Write(content); err != nil { + return "", fmt.Errorf("uploading file: writing content: %w", err) + } + if err := writer.Close(); err != nil { + return "", fmt.Errorf("uploading file: closing multipart writer: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/edge/api/rest/v2/upload", &body) + if err != nil { + return "", fmt.Errorf("uploading file: building request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + // This endpoint responds with produces=TEXT_PLAIN regardless of the request's + // Accept header's preference — asking for application/json gets a 406. + req.Header.Set("Accept", "text/plain") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("uploading file: sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("uploading file: reading response: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("uploading file: unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + + // The upload endpoint returns a plain-text artifact URI, but tolerate a + // JSON-quoted string too in case that ever changes. + var uri string + if err := json.Unmarshal(respBody, &uri); err != nil { + uri = string(bytes.Trim(respBody, "\" \n\t")) + } + if uri == "" { + return "", fmt.Errorf("uploading file: empty artifact URI in response: %s", string(respBody)) + } + + return uri, nil +} + +// EdgeConnection is the full connection resource returned by the list/get endpoints. +// It is distinct from Connection (the create/update request-response shape) because +// these read endpoints expose the connection type as a structured object. +type EdgeConnection struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + TypeId string `json:"typeId,omitempty"` + EdgeSiteId string `json:"edgeSiteId,omitempty"` + VaultId string `json:"vaultId,omitempty"` + ConnectionType *EdgeConnectionType `json:"connectionType,omitempty"` + Parameters map[string]any `json:"parameters,omitempty"` +} + +type EdgeConnectionType struct { + Id string `json:"id,omitempty"` +} + +// ListConnections lists all Edge connections via GET /edge/api/rest/v2/connections. +func ListConnections(ctx context.Context, client *http.Client) ([]EdgeConnection, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "/edge/api/rest/v2/connections", nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var connections []EdgeConnection + if err := json.Unmarshal(body, &connections); err != nil { + return nil, fmt.Errorf("failed to parse connections response: %w", err) + } + return connections, nil +} + +// GetConnection fetches a single Edge connection via GET /edge/api/rest/v2/connections/{id}. +func GetConnection(ctx context.Context, client *http.Client, id string) (*EdgeConnection, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "/edge/api/rest/v2/connections/"+id, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var conn EdgeConnection + if err := json.Unmarshal(body, &conn); err != nil { + return nil, fmt.Errorf("failed to parse connection response: %w", err) + } + return &conn, nil +} + +// DeleteConnection deletes an Edge connection via DELETE /edge/api/rest/v2/connections/{id}. +func DeleteConnection(ctx context.Context, client *http.Client, id string) error { + req, err := http.NewRequestWithContext(ctx, "DELETE", "/edge/api/rest/v2/connections/"+id, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} diff --git a/pkg/clients/edge_job_client.go b/pkg/clients/edge_job_client.go new file mode 100644 index 0000000..00197fe --- /dev/null +++ b/pkg/clients/edge_job_client.go @@ -0,0 +1,80 @@ +package clients + +import ( + "context" + "fmt" + "net/http" + "strconv" +) + +// TestConnectionResponse is the response shape returned by +// GET /edge/api/rest/v2/connections/{connectionId}/test. +type TestConnectionResponse struct { + JobID string `json:"jobId"` + Success bool `json:"success"` + Message string `json:"message,omitempty"` +} + +// TestConnection tests an Edge connection via +// GET /edge/api/rest/v2/connections/{connectionId}/test?timeoutSec=. When timeoutSec +// is nil (or <= 0), the call returns immediately with a jobId while the test runs in +// the background — poll its result with GetJobStatusLog. When timeoutSec is positive, +// the call blocks up to that many seconds and returns the final result directly. +func TestConnection(ctx context.Context, client *http.Client, connectionID string, timeoutSec *int) (*TestConnectionResponse, error) { + endpoint := "/edge/api/rest/v2/connections/" + connectionID + "/test" + if timeoutSec != nil && *timeoutSec > 0 { + endpoint += "?timeoutSec=" + strconv.Itoa(*timeoutSec) + } + + var result TestConnectionResponse + if err := getJSON(ctx, client, endpoint, "testing connection", &result); err != nil { + return nil, err + } + return &result, nil +} + +// JobStatusLog is the response shape returned by GET /edge/api/rest/v2/jobs/{id}/statusLog. +type JobStatusLog struct { + JobID string `json:"jobId"` + Status string `json:"status"` + Message string `json:"message,omitempty"` + LastUpdatedDateTime string `json:"lastUpdatedDateTime,omitempty"` +} + +// GetJobStatusLog fetches the current status of an Edge job (e.g. one started by +// TestConnection or start_ingestion's SynchronizeDatabaseMetadata) via +// GET /edge/api/rest/v2/jobs/{id}/statusLog. Terminal statuses include SUCCEEDED, +// FAILED, CANCELLED, CAPABILITY_SUCCEEDED, and CAPABILITY_FAILED. +func GetJobStatusLog(ctx context.Context, client *http.Client, jobID string) (*JobStatusLog, error) { + endpoint := "/edge/api/rest/v2/jobs/" + jobID + "/statusLog" + var result JobStatusLog + if err := getJSON(ctx, client, endpoint, "getting job status", &result); err != nil { + return nil, err + } + if result.JobID == "" { + return nil, fmt.Errorf("getting job status: empty response for job %s", jobID) + } + return &result, nil +} + +// CancelEdgeJob cancels a running Edge job via POST /edge/api/rest/v2/jobs/{id}/cancel. +func CancelEdgeJob(ctx context.Context, client *http.Client, id string) error { + req, err := http.NewRequestWithContext(ctx, "POST", "/edge/api/rest/v2/jobs/"+id+"/cancel", nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +// GetEdgeJobStatusHistory returns all status updates for an Edge job in reverse +// chronological order (most recent first) via +// GET /edge/api/rest/v2/jobs/{id}/statusLogHistory. +func GetEdgeJobStatusHistory(ctx context.Context, client *http.Client, id string) ([]JobStatusLog, error) { + endpoint := "/edge/api/rest/v2/jobs/" + id + "/statusLogHistory" + var result []JobStatusLog + if err := getJSON(ctx, client, endpoint, "getting job status history", &result); err != nil { + return nil, err + } + return result, nil +} diff --git a/pkg/clients/edge_site_client.go b/pkg/clients/edge_site_client.go new file mode 100644 index 0000000..25e2b8b --- /dev/null +++ b/pkg/clients/edge_site_client.go @@ -0,0 +1,99 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// EdgeSite is the response shape returned by the Edge site management API. +type EdgeSite struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + SiteType string `json:"siteType,omitempty"` + Status string `json:"status"` + VersionSyncStatus string `json:"versionSyncStatus,omitempty"` + InstalledVersion string `json:"installedVersion,omitempty"` + DesiredVersion string `json:"desiredVersion,omitempty"` +} + +// CapabilityType describes an available Edge capability type (e.g. "jdbc-ingestion") +// and the manifest that defines its expected install parameters. +type CapabilityType struct { + ID string `json:"id"` + // Manifest is arbitrary, capability-type-defined JSON. any (not json.RawMessage) + // is deliberate: json.RawMessage is a []byte under the hood, and chip's schema + // reflector doesn't know its custom marshaling, so it advertises an array/string + // schema for a value that's actually a JSON object at runtime. + Manifest any `json:"manifest,omitempty"` +} + +// ConnectionType describes an available Edge connection type and the manifest that +// defines its expected parameters. +type ConnectionType struct { + ID string `json:"id"` + Manifest any `json:"manifest,omitempty"` +} + +// GetEdgeSites lists all Edge sites via GET /edge/api/rest/v2/sites. +func GetEdgeSites(ctx context.Context, client *http.Client) ([]EdgeSite, error) { + var sites []EdgeSite + if err := getJSON(ctx, client, "/edge/api/rest/v2/sites", "listing edge sites", &sites); err != nil { + return nil, err + } + return sites, nil +} + +// GetCapabilityTypes lists the capability types available on an Edge site via +// GET /edge/api/rest/v2/sites/{siteId}/capabilityTypes. +func GetCapabilityTypes(ctx context.Context, client *http.Client, edgeSiteID string) ([]CapabilityType, error) { + var types []CapabilityType + endpoint := "/edge/api/rest/v2/sites/" + edgeSiteID + "/capabilityTypes" + if err := getJSON(ctx, client, endpoint, "listing capability types", &types); err != nil { + return nil, err + } + return types, nil +} + +// GetConnectionTypes lists the connection types available on an Edge site via +// GET /edge/api/rest/v2/sites/{siteId}/connectionTypes. +func GetConnectionTypes(ctx context.Context, client *http.Client, edgeSiteID string) ([]ConnectionType, error) { + var types []ConnectionType + endpoint := "/edge/api/rest/v2/sites/" + edgeSiteID + "/connectionTypes" + if err := getJSON(ctx, client, endpoint, "listing connection types", &types); err != nil { + return nil, err + } + return types, nil +} + +func getJSON(ctx context.Context, client *http.Client, endpoint, action string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("%s: building request: %w", action, err) + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("%s: sending request: %w", action, err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("%s: reading response: %w", action, err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s: unexpected status %d: %s", action, resp.StatusCode, string(body)) + } + + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("%s: decoding response: %w", action, err) + } + + return nil +} diff --git a/pkg/clients/generic_integration_client.go b/pkg/clients/generic_integration_client.go new file mode 100644 index 0000000..96d501e --- /dev/null +++ b/pkg/clients/generic_integration_client.go @@ -0,0 +1,279 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" +) + +const genericIntegrationBasePath = "/rest/catalog/1.0/genericIntegration" + +// --- Types --- + +type GenericConfiguration struct { + Id string `json:"id,omitempty"` + IngestibleId string `json:"ingestibleId,omitempty"` + Value string `json:"value,omitempty"` +} + +type SaveGenericConfigRequest struct { + Configuration string `json:"configuration"` +} + +type GenericSchedule struct { + Id int64 `json:"id,omitempty"` + CronExpression string `json:"cronExpression,omitempty"` + CronTimeZone string `json:"cronTimeZone,omitempty"` + LastRunTimeStamp int64 `json:"lastRunTimeStamp,omitempty"` + NextRunDateLongValue int64 `json:"nextRunDateLongValue,omitempty"` + CronJson string `json:"cronJson,omitempty"` + Workflow string `json:"workflow,omitempty"` +} + +type AddGenericScheduleRequest struct { + CronExpression string `json:"cronExpression"` + CronTimeZone string `json:"cronTimeZone"` + CronJson string `json:"cronJson,omitempty"` +} + +type ChangeGenericScheduleRequest struct { + CronExpression string `json:"cronExpression"` + CronTimeZone string `json:"cronTimeZone"` + CronJson string `json:"cronJson,omitempty"` +} + +type GenericJob struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + State string `json:"state,omitempty"` + Result string `json:"result,omitempty"` + Message string `json:"message,omitempty"` + ProgressPercentage float64 `json:"progressPercentage,omitempty"` + Cancelable bool `json:"cancelable,omitempty"` + StartDate int64 `json:"startDate,omitempty"` + EndDate int64 `json:"endDate,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + CreatedOn int64 `json:"createdOn,omitempty"` + UserId string `json:"userId,omitempty"` +} + +type genericScheduleParams struct { + Workflow string `url:"workflow,omitempty"` +} + +type startJobParams struct { + CloudIngestionJobId string `url:"cloudIngestionJobId,omitempty"` + Workflow string `url:"workflow,omitempty"` +} + +// --- Configuration functions --- + +func GetGenericConfig(ctx context.Context, client *http.Client, ingestibleId string) (*GenericConfiguration, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/configuration" + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var config GenericConfiguration + if err := json.Unmarshal(body, &config); err != nil { + return nil, fmt.Errorf("failed to parse config response: %w", err) + } + return &config, nil +} + +func SaveGenericConfig(ctx context.Context, client *http.Client, ingestibleId string, reqBody SaveGenericConfigRequest) (*GenericConfiguration, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/configuration" + data, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "PUT", endpoint, bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var config GenericConfiguration + if err := json.Unmarshal(body, &config); err != nil { + return nil, fmt.Errorf("failed to parse config response: %w", err) + } + return &config, nil +} + +func DeleteGenericConfig(ctx context.Context, client *http.Client, ingestibleId string) error { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/configuration" + req, err := http.NewRequestWithContext(ctx, "DELETE", endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +func GetGenericSchema(ctx context.Context, client *http.Client, ingestibleId string) (string, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/configuration/schema" + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return "", err + } + // Response is a JSON string value + var schema string + if err := json.Unmarshal(body, &schema); err != nil { + // If not a quoted string, return raw body + return string(body), nil + } + return schema, nil +} + +// --- Schedule functions --- + +func GetGenericSchedule(ctx context.Context, client *http.Client, ingestibleId string) (*GenericSchedule, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/schedule" + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var schedule GenericSchedule + if err := json.Unmarshal(body, &schedule); err != nil { + return nil, fmt.Errorf("failed to parse schedule response: %w", err) + } + return &schedule, nil +} + +func AddGenericSchedule(ctx context.Context, client *http.Client, ingestibleId string, workflow string, reqBody AddGenericScheduleRequest) (*GenericSchedule, error) { + params := genericScheduleParams{Workflow: workflow} + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/schedule", params) + if err != nil { + return nil, fmt.Errorf("failed to build endpoint: %w", err) + } + return doGenericScheduleRequest(ctx, client, "POST", endpoint, reqBody) +} + +func UpdateGenericSchedule(ctx context.Context, client *http.Client, ingestibleId string, workflow string, reqBody ChangeGenericScheduleRequest) (*GenericSchedule, error) { + params := genericScheduleParams{Workflow: workflow} + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/schedule", params) + if err != nil { + return nil, fmt.Errorf("failed to build endpoint: %w", err) + } + return doGenericScheduleRequest(ctx, client, "PUT", endpoint, reqBody) +} + +func DeleteGenericSchedule(ctx context.Context, client *http.Client, ingestibleId string, workflow string) error { + params := genericScheduleParams{Workflow: workflow} + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/schedule", params) + if err != nil { + return fmt.Errorf("failed to build endpoint: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "DELETE", endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +func GetAllGenericSchedules(ctx context.Context, client *http.Client, ingestibleId string) (*GenericSchedule, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/schedules" + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var schedule GenericSchedule + if err := json.Unmarshal(body, &schedule); err != nil { + return nil, fmt.Errorf("failed to parse schedules response: %w", err) + } + return &schedule, nil +} + +func doGenericScheduleRequest(ctx context.Context, client *http.Client, method, endpoint string, reqBody any) (*GenericSchedule, error) { + data, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var schedule GenericSchedule + if err := json.Unmarshal(body, &schedule); err != nil { + return nil, fmt.Errorf("failed to parse schedule response: %w", err) + } + return &schedule, nil +} + +// --- Job functions --- + +func CancelGenericJob(ctx context.Context, client *http.Client, ingestibleId string, workflow string) error { + params := genericScheduleParams{Workflow: workflow} + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/cancel", params) + if err != nil { + return fmt.Errorf("failed to build endpoint: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "DELETE", endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +func StartGenericJob(ctx context.Context, client *http.Client, ingestibleId string, cloudIngestionJobId string, workflow string, runtimeArguments string) (*GenericJob, error) { + params := startJobParams{ + CloudIngestionJobId: cloudIngestionJobId, + Workflow: workflow, + } + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/run", params) + if err != nil { + return nil, fmt.Errorf("failed to build endpoint: %w", err) + } + + var bodyReader *bytes.Reader + if runtimeArguments != "" { + data, err := json.Marshal(runtimeArguments) + if err != nil { + return nil, fmt.Errorf("failed to marshal runtime arguments: %w", err) + } + bodyReader = bytes.NewReader(data) + } else { + bodyReader = bytes.NewReader([]byte{}) + } + + req, err := http.NewRequestWithContext(ctx, "POST", endpoint, bodyReader) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var job GenericJob + if err := json.Unmarshal(body, &job); err != nil { + return nil, fmt.Errorf("failed to parse job response: %w", err) + } + return &job, nil +} diff --git a/pkg/clients/prepare_create_asset_client.go b/pkg/clients/prepare_create_asset_client.go index 0d9cf83..1ff9e15 100644 --- a/pkg/clients/prepare_create_asset_client.go +++ b/pkg/clients/prepare_create_asset_client.go @@ -61,17 +61,16 @@ type PrepareCreateDomainListResponse struct { Total int `json:"total"` } - // PrepareCreateAttributeType represents an attribute type with full schema. type PrepareCreateAttributeType struct { - ID string `json:"id"` - Name string `json:"name"` - Kind string `json:"kind"` - Required bool `json:"required"` - Constraints *PrepareCreateConstraints `json:"constraints,omitempty"` - AllowedValues []string `json:"allowedValues,omitempty"` - Direction string `json:"direction,omitempty"` - TargetAssetType *PrepareCreateAssetType `json:"targetAssetType,omitempty"` + ID string `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` + Required bool `json:"required"` + Constraints *PrepareCreateConstraints `json:"constraints,omitempty"` + AllowedValues []string `json:"allowedValues,omitempty"` + Direction string `json:"direction,omitempty"` + TargetAssetType *PrepareCreateAssetType `json:"targetAssetType,omitempty"` } // PrepareCreateConstraints represents attribute validation constraints. diff --git a/pkg/skills/files/collibra/discovery/SKILL.md b/pkg/skills/files/collibra/discovery/SKILL.md index ec5eb87..8a91067 100644 --- a/pkg/skills/files/collibra/discovery/SKILL.md +++ b/pkg/skills/files/collibra/discovery/SKILL.md @@ -1,6 +1,6 @@ --- description: Pick between natural-language semantic search and keyword/filter search to find Collibra assets and terms. -related: collibra/lineage, collibra/asset-create +related: collibra/lineage, collibra/asset-create, collibra/jdbc-ingestion --- # Discovery — finding assets and business terms diff --git a/pkg/skills/files/collibra/etl-integration/SKILL.md b/pkg/skills/files/collibra/etl-integration/SKILL.md new file mode 100644 index 0000000..71d17b3 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/SKILL.md @@ -0,0 +1,173 @@ +--- +description: Set up, configure, schedule, and run Edge ETL integrations (metadata sync capabilities like Dataplex, Databricks Unity Catalog, Purview, Sigma) end to end. +related: collibra/jdbc-ingestion, collibra/discovery +--- + +# ETL integrations — connect, configure, schedule, run + +An ETL integration is an Edge **capability**: a small app running in the Edge Kubernetes +cluster that extracts metadata from a third-party system and loads it into the Collibra +Data Governance Platform (DGC). Setup spans two systems — **Edge** (connection + +capability) and **DGC catalog** (the generic config + schedule/run). No single tool does +the whole thing; calling them out of order is the most common way this fails. + +## First: confirm this is the right ingestion path + +Many data sources support **more than one** ingestion path — most commonly a native +**ETL integration** (this skill) *and* generic **JDBC** (`collibra/jdbc-ingestion`). Which +one to use is a real decision, not a default. + +**Whenever the user asks to create an integration, before doing anything else, determine +which ingestion type it should be** — don't assume ETL just because this skill is loaded: + +1. Identify the data source. +2. Check whether more than one path supports it (`references/index.md` lists the data + sources with a native ETL integration; anything reachable over a JDBC driver can also go + through `collibra/jdbc-ingestion`). +3. **If more than one path is possible, ask the user which they want** before starting. + Only proceed with this ETL skill once the user has chosen it (or it's the only supported + path). + +## The three configuration layers (merged at run time) + +When a run starts, DGC's catalog module merges three parameter sets into one and sends +them to the Edge site to start the capability's workflow; it also creates a **Job** in +DGC to track progress and, on completion, records status metrics and a report. + +1. **Edge connection** — credentials/endpoint for the third-party system. + `edge_create_connection`. +2. **Edge capability** — the integration instance and its capability parameters (most are + shared across capabilities, some are specific). `edge_create_capability`. **Its id is + the `ingestibleId`** you use for every catalog step below. +3. **Generic (ETL) config** — the sync configuration specific to the integration. + `catalog_etl_save_generic_config`. + +## The full sequence + +1. **`edge_list_sites`** → the `edgeSiteId` you'll reference below. +2. **`edge_list_capability_types`** (pass a `query`) → find the connection type and + capability type for your integration and read their parameter manifests. Don't guess + parameter names — read them here. +3. **`edge_create_connection`** → the credentials for the third-party system. +4. **`edge_create_capability`** → `typeId` = the capability type from step 2; + `parameters.connection` = the connection id from step 3. **The returned capability id + is your `ingestibleId`.** +5. **`catalog_etl_get_schema(ingestibleId)`** → the JSON schema for the generic config. + **Always fetch this and shape the config to match it — never hardcode or guess field + names.** (See "Never hardcode the config schema".) +5a. **Resolve DGC prerequisites** — nearly every integration's generic config requires at + least a `domain` UUID; many also require a `community` or a `system` (System asset) UUID. + **Before building the config, ask the user** whether they want to use an existing + domain/System or create new ones. (See "DGC prerequisites".) +6. **`catalog_etl_save_generic_config(ingestibleId, configuration)`** → the config JSON, + conforming to the schema from step 5. Read it back any time with + **`catalog_etl_get_config`**. +7. Run or schedule: + - **`catalog_etl_start_job(ingestibleId[, workflow])`** — run immediately. + - **`catalog_etl_add_schedule(ingestibleId, cronExpression, cronTimeZone[, workflow])`** + — schedule for later. Inspect/update/remove with + `catalog_etl_get_schedule` / `catalog_etl_get_all_schedules` / + `catalog_etl_update_schedule` / `catalog_etl_delete_schedule`. +8. **Track:** poll the DGC job with **`get_job_status`** (find recent runs with + `jobs_find`); **`list_integrations`** gives the schedule + last/next run overview and is + the entry point for "show me all integrations" questions. Stop a running sync with + `catalog_etl_cancel_job`. + +## Workflows + +A capability has one or more workflows. **Inbound is the default** — omit the `workflow` +argument to target it. Some integrations expose multiple inbound workflows (e.g. different +entry types) and some an outbound workflow (DGC → third party). `catalog_etl_start_job`, +`catalog_etl_add_schedule`, `catalog_etl_update_schedule`, `catalog_etl_delete_schedule`, +and `catalog_etl_cancel_job` all take an optional `workflow` — omit it for the default +inbound, or set it to target a specific one. The **generic config is per-integration** +(one config via `get_schema`/`save_generic_config`/`get_config`), not per-workflow. + +## DGC prerequisites: domain, community, and System asset + +Most integrations require one or more of these DGC objects to already exist before +`catalog_etl_save_generic_config` can succeed. The exact required fields are in the schema +from `catalog_etl_get_schema` — but the pattern is consistent across integrations: + +| Field in generic config | DGC object | Notes | +|---|---|---| +| `domain` | Domain | Where ingested assets land. Almost universal. | +| `community` | Community | Required by some integrations (e.g. Dataplex CloudSQL, Purview). | +| `system` | System asset | Target System asset for the ingested data. Required by several integrations. | + +**Always ask the user before building the config:** + +> "Do you already have a domain/System you want to use, or should I create new ones?" + +### Using existing objects + +- Resolve a domain or System UUID with **`search_asset_keyword`** (filter by asset type + `"Domain"` or `"System"`) or **`prepare_create_asset`** (lists available domains and their + communities). + +### Creating new objects + +Follow this order — each step is a prerequisite for the next: + +1. **`create_community`** — if no suitable community exists yet. +2. **`create_domain`** — within the community. Use **`find_domain_types`** to resolve the + domain type name (e.g. `"Technology Asset Domain"`) to a `typeId` UUID first. +3. **`create_asset`** — to create a System asset, if required. Use **`prepare_create_asset`** + to resolve the `"System"` asset type UUID and pick the target domain before calling. + +## Never hardcode the config schema + +Each integration's generic-config schema lives in that integration's own repo (as a +react-json-schema-form data-schema) and evolves with the integration's version. This skill +deliberately **does not embed those schemas** — always call `catalog_etl_get_schema` at run +time. That endpoint returns the very schema DGC uses to validate +`catalog_etl_save_generic_config`, so it can never be stale or wrong for the customer's +version. Likewise, read connection/capability parameter names from +`edge_list_capability_types`, not from memory. The per-integration references capture the +stable conceptual detail (purpose, auth options, flows, gotchas) — not exact field lists. + +## Per-integration references + +For a specific integration, load its reference for auth types, flows, the capability/ +connection `typeId`s, and notable config nuances: `references/.md` (e.g. +`references/dataplex.md`). `references/index.md` lists them all. These **complement, not +replace,** the live schema from `catalog_etl_get_schema`. + +## Distinct from jdbc-ingestion + +`collibra/jdbc-ingestion` is a specialized flow: it registers a Database asset via +`register_database`, configures schemas via `configure_database_schemas`, and runs via +`start_ingestion` (DGC's `synchronizeMetadata`), **not** `catalog_etl_start_job`. Use +that skill for JDBC database ingestion; use **this** skill for +the generic/ETL sync capabilities (Dataplex, Databricks Unity Catalog, Purview, Sigma, +ThoughtSpot, …). + +## Not controllable via chip (yet) + +The cloud **object-storage** crawlers are **not** ETL integrations in the sense above and +**cannot be configured through chip** — they use different endpoints and a different +configuration model than the `catalog_etl_*` / edge tools here: + +- **AWS S3** (`s3-synchronization`) +- **Azure Data Lake Storage / ADLS** (`adls-synchronization`) +- **Google Cloud Storage / GCS** (`gcs-synchronization`) + +If a user asks to set up, configure, or run one of these, tell them it isn't supported via +chip yet and must be configured in the Collibra/Edge UI — do not attempt it with the tools +in this skill. + +## Hard rules + +1. **Fetch the config schema (`catalog_etl_get_schema`) before building the generic + config** — never guess field names or reuse a stale schema. +2. **Read connection/capability parameters from `edge_list_capability_types`,** not memory. +3. **Confirm scope with the user** (which projects/datasets/filters) — don't default to + "ingest everything." +4. **Two job-id spaces:** Edge jobs every started edge capability produce one → `edge_get_job_status`; + DGC jobs (from `catalog_etl_start_job`) → `get_job_status`. Never cross them. ETL capabilities utilize DGC jobs. Don't check Edge jobs unless directly asked +5. **Inbound only** unless the user explicitly asks to configure an outbound flow. +6. **Before building the generic config, ask the user** whether to use existing DGC + domain/community/System assets or create new ones — never assume they exist. See + "DGC prerequisites". +7. To fill in parameters ask user questions and before saving them confirm with the user. +8. Before creating capability confirm capability parameters with the user \ No newline at end of file diff --git a/pkg/skills/files/collibra/etl-integration/references/azure-foundry.md b/pkg/skills/files/collibra/etl-integration/references/azure-foundry.md new file mode 100644 index 0000000..b76f8da --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/azure-foundry.md @@ -0,0 +1,67 @@ +# Azure AI Foundry (`azure-foundry-synchronization`) + +Extracts AI asset metadata from Microsoft Azure AI Foundry (Azure Machine Learning +Workspaces, Azure Cognitive Services, Azure OpenAI) and ingests it into DGC — covering +AI projects, model deployments, AI agents, fine-tuned models, training files, AI +endpoints, and AI monitors. `capability-type` (the `typeId` for +`edge_create_capability`): **`azure-foundry-synchronization`**. + +**Docs:** [Steps overview: Integrate Azure AI Foundry via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/AzureAIFoundryModel/co_edge-integrate-azure-ai-foundry-steps.htm) + +**Connection type:** Azure. Auth methods (set on `edge_create_connection`): +- **Azure Service Principal (Client Credentials Flow)** — `clientId`, `clientSecret`, + `tenantId`. This is the only supported authentication method. + +The Service Principal needs `Reader` on the subscription plus `AzureML Data Scientist` +(or `Reader`) on each ML Workspace and `Cognitive Services Contributor` (or `Reader`) on +each Cognitive Services account. + +**Workflows:** inbound only, `main` (default — omit `workflow`). There is no outbound +(DGC → Azure) flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — UUID of the existing DGC domain where all assets will be created. + All workspaces share one domain; per-workspace domain routing is not supported. +- `workspaceNames` (required) — array of workspace identifiers in + `"resourceGroup/workspaceName"` format for ML workspaces, or + `"resourceGroup/accountName/projectName"` for Azure AI Foundry projects. At least one + entry is required. +- `associateAIUseCasesWithAIModels` (default `true`) — when true, creates `implementedBy` + relations between existing Collibra AI Use Case assets and ingested AI Model Version / + Base Model assets. +- `createFileAssets` (default `true`) — when true, ingests file metadata linked to AI + agents (training files, vector store files) and fine-tuning jobs as `File` and + `FileContainer` assets. +- `customAiMetricsMappings` (optional) — maps Azure Monitor metric names (e.g. + `TokenTransaction`, `RequestsPerMinute`) to Collibra attribute type UUIDs on + `AIModelDeployment` assets. Each entry: `{"from": "MetricName", "to": ""}`. The + UUID must exist as an attribute type in DGC or the job fails at validation before any + Azure data is fetched. +- `assetDeletionMode` (Edge capability parameter, default `Soft`) — `Soft` archives + missing assets; `Hard` permanently deletes them. +- `subscriptionId` (Edge capability parameter, required) — Azure Subscription ID + containing the AI Foundry workspaces. + +## Gotchas +- **Hub workspaces are expanded automatically.** If a workspace listed in `workspaceNames` + has `kind` containing `"hub"`, all child project workspaces are discovered and ingested + automatically — you do not need to list child projects individually. +- **All assets land in one domain.** There is no per-workspace or per-resource-group + domain routing. Choose the target domain carefully before the first run. +- **Each run is a full sync.** There is no incremental/delta mode. Every run fetches all + listed workspaces from scratch. +- **DGC version gates features.** Several asset types only appear on newer DGC versions: + `AIBaseModel`, `AIEndpoint`, `AIMonitor`, and `Vendor` assets require DGC 2026-03+ + (or 2026-04+ for Vendor). Fine-tuned model assets require DGC 2025-09+. The capability + enforces these limits at startup and skips unsupported assets rather than failing. +- **Fine-tuned model deployments auto-delete in Azure.** Azure removes inactive fine-tuned + model deployments after 15 days. The next sync will archive (Soft) or delete (Hard) the + corresponding `AIModelDeployment` asset in DGC. Schedule runs at least every 15 days to + keep DGC in sync. +- **`customAiMetricsMappings` UUIDs are validated before any Azure fetch.** If any `to` + UUID is not a valid attribute type in DGC, the job fails immediately with + `InvalidParametersException` before contacting Azure. +- **Metric attributes are not populated if Azure Monitor has no data.** If a deployment + has never received traffic, mapped metric attributes will remain unset. +- **Workspace names must match exactly.** An unrecognised workspace name in `workspaceNames` + throws `InvalidParametersException` before any data is fetched. diff --git a/pkg/skills/files/collibra/etl-integration/references/azureml.md b/pkg/skills/files/collibra/etl-integration/references/azureml.md new file mode 100644 index 0000000..50c9da2 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/azureml.md @@ -0,0 +1,24 @@ +# Azure Machine Learning (`azureml-synchronization`) + +Extracts metadata from Microsoft Azure Machine Learning workspaces and ingests it into Collibra DGC as **Azure AI Model Version** (`AzureAIModel`) assets, including performance metrics fetched from the embedded MLflow tracking server. + +**Docs:** [About the Azure ML integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/AzureAIModel/co_about-azure-ai.htm) + +**Connection type:** Azure (generic). Auth method (set on `edge_create_connection`): +- **Service Principal** — OAuth 2.0 Client Credentials flow; requires `clientId`, `clientSecret`, and `tenantId` for the Azure AD application. + +**Workflows:** single inbound workflow `main` (omit `workflow` argument — there is no outbound flow). + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — target Collibra Technology Asset Domain where `AzureAIModel` assets are created. +- `resourceGroupName` (required) — Azure resource group containing the workspace. +- `workspaceName` (required) — Azure ML workspace to synchronize; must be unique within the resource group. +- `customAiMetricsMappings` (optional) — maps MLflow metric keys (e.g., `f1_score_macro`) to Collibra attribute type IDs on the `AzureAIModel` asset type. The only metric auto-populated without a mapping is `accuracy` → `ModelAccuracy`; all other standard attributes (`ModelPrecision`, `MeanSquaredError`, `MeanAbsoluteError`) require an explicit mapping entry here. + +## Gotchas +- **Latest version only**: one `AzureAIModel` asset is created per model container, representing only the latest registered version. +- **AutoML models silently skipped**: models whose names match the AutoML child-run pattern are excluded without warning; if a workspace appears empty, verify it contains user-registered (non-AutoML) models. +- **Custom metric validation is eager**: if a `customAiMetricsMappings` entry references an attribute type not already assigned to the `AzureAIModel` asset type in Collibra, the synchronization aborts before fetching any data. Pre-assign the attribute types first. +- **No metrics when no job run**: models that were registered without an associated MLflow job run ID produce assets with name, version, and description only — no metric attributes are populated. +- **`InitiatingUserInSource` requires DGP 2026.07+**: on earlier platform versions this attribute is silently omitted even when present in the source. +- **Azure permissions**: the service principal needs `Microsoft.MachineLearningServices/workspaces/models/versions/read` and `workspaces/jobs/read`. The built-in **AzureML Data Scientist** role covers both; a plain **Reader** role retrieves models but no MLflow metrics. diff --git a/pkg/skills/files/collibra/etl-integration/references/bedrock.md b/pkg/skills/files/collibra/etl-integration/references/bedrock.md new file mode 100644 index 0000000..61a57c5 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/bedrock.md @@ -0,0 +1,42 @@ +# AWS Bedrock AI (`bedrock-synchronization`) + +Extracts AI metadata from AWS Bedrock (foundation models, custom/imported model +versions, deployments, endpoints, agents, vendors, and — when data ingestion is +enabled — the S3 training datasets and invocation-log output locations) and ingests +it into DGC. `capability-type` (the `typeId` for `edge_create_capability`): +**`bedrock-synchronization`**. + +**Docs:** [About the AWS Bedrock AI integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/AWSBedrock/co_about-aws-bedrock-ai.htm) + +**Connection type:** AWS (`family: generic`). Auth methods (set on `edge_create_connection` +via the `authentication` property): +- **IAM static credentials** — `authentication = IAM` plus `aws_access_key_id` / + `aws_secret_access_key`; for Edge running outside AWS. +- **Instance / Pod role** — any other `authentication` value; keys left blank, resolves + the EC2 instance role or EKS IRSA role via the AWS default credentials chain. + +**Workflows:** inbound only, `main` (default — omit `workflow`). There is no outbound +(DGC → Bedrock) flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — target domain UUID; all catalogued assets are placed here. +- `regions` — AWS regions to scan; if empty, every region where Bedrock is generally + available is scanned (only regions actually offering Bedrock return results). +- `ingestAiInputAndOutput` (default true) — also ingest S3 File/File Container assets for + custom-model training data and deployment invocation-log output; when false, only model + metadata is ingested (cheaper, fewer AWS calls). +- `customAiMetricsMappings` — map a custom Bedrock training/validation metric name to a + Collibra AI Model attribute type; without mappings no metric values are persisted. + +## Gotchas +- IAM mode requires **both** keys — the run fails with `InvalidParametersException` if + either is missing while `authentication = IAM`. +- Per-region resilience: a region returning 4xx is marked "without access" and a region + where Bedrock isn't deployed (`UnknownHostException`) is marked "without Bedrock"; the + run continues and reports **Completed with Errors** if at least one region returned data, + or **Failed** only if every region was rejected. +- `ingestAiInputAndOutput` triggers extra AWS calls (`GetCustomModel`, + `GetModelInvocationLoggingConfiguration`) — CloudWatch-only logging is skipped, and + missing per-model permissions degrade gracefully (model imported without metrics). +- `save-input-metadata` uploads a ZIP of raw AWS responses to DGC; enable only at Collibra + Support's request, as it snapshots all metadata visible to the AWS principal. diff --git a/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md b/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md new file mode 100644 index 0000000..8ad4655 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md @@ -0,0 +1,35 @@ +# Technical Lineage for Databricks Unity Catalog (`databricks-lineage`) + +Extracts data lineage from the Databricks Unity Catalog lineage system tables and produces column-level technical lineage in DGC, stitched to existing Databricks assets. +`capability-type` (the `typeId` for `edge_create_capability`): **`databricks-lineage`**. + +**Docs:** [Add the technical lineage for Databricks Unity Catalog capability](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/DatabricksUnityCatalog/ta_databricks-add-capability.htm) + +**Connection type:** Databricks (requires `workspaceUrl`). Auth methods (set via `authType` on `edge_create_connection`): +- **Personal Access Token** — `accessToken`. +- **OAuth** — `clientId` + `clientSecret`. +- **Microsoft Entra ID** — `clientId` + `clientSecret` + `tenantId`. + +**Workflows:** inbound is `main` (default — omit `workflow`). No outbound flow; instead, running with the `active` capability parameter set to `false` for a `sourceId` removes that source's lineage from the Techlin server. + +## Capability parameters (set on `edge_create_capability`) +- `httpPath` (required) — HTTP path of the SQL warehouse / compute resource (must start with `sql`). +- `sourceId` (required) — unique lineage source id; also the key for the exclusion flow. +- `processingLevel` — `Analyze` (load + analyze) or `Sync` (also synchronize to DGC). +- `timeFrame` — lookback window in days (default 365). +- **`customParameters` (mandatory)** — a user-defined list that must include the Techlin server coordinates: + - `techlinHost` — the Techlin/Data Lineage server host URL for this environment (e.g. `https://techlin-.cp.collibra-ops.com`). + - `techlinKey` — the Techlin API key. +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `system` (required) — target System asset the ingested objects are stitched to. +- `includeFilters` / `excludeFilters` — scope by `Workspace > Catalog > Schema` (wildcards `*`/`?`; exclude wins). +- `catalog` — read lineage from views in a custom catalog instead of the `system` catalog. +- `sqlSourcesLimit` / `ingestSqlSources` — include SQL transformation texts, capped per relation. +- `ingestExternalLocations`, `ingestVolumes`, `ingestEntities` — extend coverage to external locations, volumes, and notebook/entity transformations. + +## Gotchas +- This capability does **not** create the Databricks assets — they must already exist in DGC (ingested by the separate `databricks-edge-capability`) for stitching to work; stitching matches on the exact full path `(System >) Database > Schema > Table > Column`. +- Set `useCollibrasystemName` when the same paths exist across environments (e.g. PROD/DEV), otherwise stitching is non-deterministic; changing it requires re-analyzing all sources. +- Requires read permissions on the Unity Catalog lineage system tables (`system.access.column_lineage`, `system.access.table_lineage`, `system.query.history`) plus compute attach rights. +- Lineage retention is fixed at 365 days by Databricks; large datasets rely on the default `scalable` query strategy (tune via `jvm-args`). +- **`techlinHost`/`techlinKey` custom parameters are mandatory** — set them before the first run. See the `customParameters` entry under "Capability parameters". diff --git a/pkg/skills/files/collibra/etl-integration/references/databricks-uc.md b/pkg/skills/files/collibra/etl-integration/references/databricks-uc.md new file mode 100644 index 0000000..285d8d3 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/databricks-uc.md @@ -0,0 +1,47 @@ +# Databricks Unity Catalog (`databricks-edge-capability`) + +Extracts metadata from Databricks Unity Catalog (catalogs, schemas, tables, columns, volumes, AI models/deployments/endpoints) and ingests it into DGC. The `typeId` for `edge_create_capability` is **`databricks-edge-capability`**. + +**Docs:** [About the Databricks Unity Catalog integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/Databricks/co_about-databricks-integration.htm) + +**Connection type:** Databricks. Auth methods (set on `edge_create_connection`): +- **Personal Access Token** — `accessToken` field. +- **OAuth** — `clientId` + `clientSecret`. +- **Microsoft Entra ID** — `clientId` + `clientSecret` + `tenantId`. + +**Workflows:** +- `main` (default) — combined inbound; flow type is chosen by `ingestionType` in the generic config. +- `ai` — dedicated AI asset inbound (models, deployments, endpoints, agents, monitors; DGC 2026.03+). +- `ai-monitoring` — reads MLflow trace metrics via JDBC and POSTs them to the Collibra AI Governance API; does **not** create or modify DGC assets. +- `outbound` — writes DGC attribute changes back to Databricks Unity Catalog as tags. + +## Ingestion modes (selected in the generic config, NOT via the workflow arg) + +Applies when using the `main` (default) workflow via the `ingestionType` config field: + +- **Metadata ingestion** — standard catalog/schema/table/column structure, JDBC-compatible with other ingestion capabilities. Requires `system`. +- **AI model ingestion** (Deprecated) — ingests AI base models, versions, deployments, endpoints, agents, monitors. Requires `domain`. +- **Metadata and AI model ingestion** (Deprecated) — combined; requires both `system` and `domain`. + +When using the dedicated `ai` workflow, `ingestionType` is not used — the workflow name itself determines the flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) + +- `system` — required for metadata flows; the System asset in DGC that becomes the root for ingested catalog/schema/table assets. +- `domain` — required for AI model flows; the DGC domain where AI assets land. +- `domainIncludeMappings` / `domainExcludeMappings` — scope ingestion to specific catalogs, schemas, or tables using `catalog > schema > table` patterns (wildcards `*` and `?` supported). Empty = ingest everything the connection can see. +- `version` — domain mapping resolution version. `V0` (default): unmatched schemas inherit the catalog domain. `V1`: unmatched schemas are excluded. New configurations should use `V1`. +- `ingestVolumes` — when enabled, also ingests Databricks Volume assets (DatabricksVolume, Directory, File) for each schema. Requires the Databricks Files API to be enabled on the workspace. +- `tagAttributeMappings` — routes Unity Catalog tag values to specific DGC attributes in addition to (or instead of) the default `Source Tags` attribute. +- `extensiblePropertiesMapping` — maps Databricks system properties (e.g. `table_type`, `created_at`, `metastore_id`) and custom table parameters to DGC attributes. +- `ingestInputDatasetsAndDeploymentOutput` (AI flow) — enables ingestion of MLflow training datasets and inference table output linked to model versions/deployments. + +## Gotchas + +- **JDBC connection is required for tag fetching.** A separate `jdbc-connection` capability parameter must reference a JDBC-type Edge connection. Without it, Unity Catalog tags are not fetched and `Source Tags` will be empty. The same JDBC connection also enables Profiling and Classification. +- **Domain cannot be changed for existing assets.** Once a catalog (Database) or schema asset has been ingested, its domain is locked — the engine looks up existing assets by name and reuses their domain regardless of the current mapping. Get domain mappings right before the first run. +- **Do not configure AI ingestion in both tabs.** Providing AI parameters in both the "Metadata Inbound" tab and the dedicated "AI ingestion" tab causes synchronization to fail on DGC < 2026.04. +- **Governed tags in Databricks only accept predefined values.** The outbound flow silently skips tag writes whose value is not permitted by the tag policy; the DGC attribute is updated but the Databricks tag stays unchanged. Check outbound logs for errors when mapping governed tags. +- **Volumes ingestion requires a Databricks beta flag.** The Files API (`enable_experimental_files_api_client = True`) must be enabled on the workspace before `ingestVolumes` will work. +- **`ai-monitoring` produces no DGC catalog assets.** It writes only to the AI Governance API (`/rest/aiGovernance/v1/metrics/bulk`). There is no synchronization report entry; success/failure is reported as an Edge job status only. +- **`stopComputeResource` shuts down the compute cluster after tag extraction.** Only relevant when `httpPath` is set. Enable with caution in shared-cluster environments. diff --git a/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md b/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md new file mode 100644 index 0000000..4baa975 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md @@ -0,0 +1,40 @@ +# Technical Lineage for GCP (`dataplex-lineage-synchronization`) + +Extracts technical lineage from Google Dataplex (Data Lineage API) plus BigQuery / GCS and pushes it to Collibra Data Lineage (Techlin), producing table- and column-level technical lineage in DGP. This is a lineage capability — it does **not** ingest table/column assets; those must already exist in DGP for stitching to work. +`capability-type` (the `typeId` for `edge_create_capability`): **`dataplex-lineage-synchronization`**. + +**Docs:** [Synchronize technical lineage for Google Dataplex](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/GoogleDataplex/ta_dataplex-sync-techlin.htm) + +**Connection type:** GCP. Auth methods (set on `edge_create_connection`): +- **Service Account** — full JSON key (`gcp_service_account_credentials_json`). +- **Workload Identity Federation (WIF)** — federated identity; JSON only for file-based WIF. +- **WIF using GKE** — when the Edge site runs in GKE; uses the site's federated identity. + +**Workflows:** inbound only (default — omit `workflow`). No outbound flow. Setting the capability param `active=false` for a `sourceId` triggers removal of that source's lineage from Techlin instead of generating any. + +## Ingestion modes (selected in the generic config, NOT via the workflow arg) +- **Table lineage** (`lineageType` default) — table-level lineage; supports `includeFilters` / `excludeFilters`. +- **Column lineage** — column-level lineage; requires `gcsBucket` and the (preview) Export Metadata API. Note: stitching is not supported at table level, only column level. + +## Capability parameters (set on `edge_create_capability`) +- `active` — set `false` for a `sourceId` to remove that source's lineage from Techlin (there is no outbound flow). +- `useCollibrasystemName` — see gotchas; enable when identical paths exist across environments. +- **`customParameters` (mandatory)** — a user-defined list that must include the Techlin server coordinates: + - `techlinHost` — the Techlin/Data Lineage server host URL for this environment (e.g. `https://techlin-.cp.collibra-ops.com`). + - `techlinKey` — the Techlin API key. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `system` (required) — target System asset the ingested objects are stitched to. +- `projects` — GCP project IDs where Dataplex lineage is enabled (required for WIF; else derived from the SA key). +- `locations` — GCP regions to harvest from; defaults to the supported Dataplex regions. +- `lineageType` — Table vs Column lineage (see modes above). +- `gcsBucket` — required for Column lineage; path to the GCS bucket holding exported lineage. +- `includeFilters` / `excludeFilters` — Table-lineage only; patterns `project > location > @bigquery > dataset > table` with `*`/`?` wildcards; exclude wins over include. +- `sqlExtractionMethod` — `api` (default, one Get Job call per process) vs `query` (batched `INFORMATION_SCHEMA.JOBS_BY_PROJECT`). + +## Gotchas +- Stitching matches on the **exact full path** `(System >) Database > Schema > Table > Column`; assets must be pre-ingested (e.g. via BigQuery JDBC) or nothing stitches. GCS-file stitching is not supported. +- Enable `useCollibrasystemName` (capability param) when identical paths exist across environments (PROD/DEV) — otherwise stitching is non-deterministic. Changing it requires re-analyzing all sources. +- Dataplex retains lineage for only 30 days; older lineage is permanently gone, so runs must be scheduled recurringly. +- Column lineage depends on the preview `exportMetadata` API, which must be enabled by Google on request. +- **`techlinHost`/`techlinKey` custom parameters are mandatory** — set them before the first run. See the `customParameters` entry under "Capability parameters". diff --git a/pkg/skills/files/collibra/etl-integration/references/dataplex.md b/pkg/skills/files/collibra/etl-integration/references/dataplex.md new file mode 100644 index 0000000..52cdad4 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/dataplex.md @@ -0,0 +1,36 @@ +# Dataplex / Google Knowledge Catalog (`dataplex-synchronization`) + +Extracts metadata from Google Dataplex / Knowledge Catalog (formerly Dataplex Universal +Catalog) and ingests it into DGC. `capability-type` (the `typeId` for +`edge_create_capability`): **`dataplex-synchronization`**. + +**Docs:** [About Google Dataplex ingestion via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/GCS/Dataplex/co_about-dataplex.htm) + +**Connection type:** GCP. Auth methods (set on `edge_create_connection`): +- **Service Account** — full JSON key (`gcp_service_account_credentials_json`). +- **Workload Identity Federation (WIF)** — federated identity; JSON only for file-based WIF. +- **WIF using GKE** — when the Edge site runs in GKE; uses the site's federated identity. + +**Workflows:** inbound is `main` (default — omit `workflow`). Also has an outbound flow +(DGC → Dataplex). + +## Ingestion modes (selected in the generic config, NOT via the workflow arg) +- **DataSourceType**: `BigQuery` (default) or `CloudSQL` (MySQL / PostgreSQL / SQL Server). +- For BigQuery, **ingestionType** picks the ETL flow: **Knowledge Catalog** ingestion + (JDBC-compatible db/schema/table/column structure) or legacy **Dataplex** ingestion + (older non-JDBC APIs, still supported). + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `system` (required) — target System asset; `community` required for CloudSQL. +- `projects` / `locations` (optional) — scope ingestion; empty = all the connection can see. +- `domainIncludeMappings` / `domainExcludeMappings` + `domainMappingVersion` (V0 keeps an + existing asset's domain; V1 lets the mapping override it). +- `aspectMappings` (Knowledge Catalog) / `customLabelMappings` (legacy Dataplex) — map + source aspects/labels to DGC attributes. +- `columnsIngestionMode` — parent-only / parent+nested / flattened column structures. + +## Gotchas +- The BigQuery-vs-CloudSQL and Knowledge-Catalog-vs-legacy choice lives in the **config** + (`DataSourceType`/`ingestionType`), not the Edge `workflow` argument. +- An optional `jdbc-connection` capability parameter only enables profiling/classification + of ingested data — it is not the ingestion path itself. diff --git a/pkg/skills/files/collibra/etl-integration/references/datazone.md b/pkg/skills/files/collibra/etl-integration/references/datazone.md new file mode 100644 index 0000000..7a7d7fd --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/datazone.md @@ -0,0 +1,27 @@ +# AWS DataZone / SageMaker Unified Studio (`datazone-synchronization`) + +Extracts Redshift or Glue table/view metadata published in AWS SageMaker Unified Studio and ingests it into DGC as a System → Database → Schema → Table/View → Column hierarchy. The `typeId` for `edge_create_capability` is **`datazone-synchronization`**. + +**Docs:** [About the Amazon SageMaker Unified Studio integration](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/AwsSagemakeUnifiedStudioDataZone/co_about-aws-sagemaker-unified-studio.htm) + +**Connection type:** AWS (generic family). Auth methods (set on `edge_create_connection`): +- **IAM** — programmatic user credentials (`aws_access_key_id` + `aws_secret_access_key`). This is the only supported method. + +**Workflows:** inbound is `main` (default — omit `workflow`). Also has an outbound flow (DGC → SageMaker Unified Studio). + +## Ingestion modes (selected in the generic config, NOT via the workflow arg) +- **datasource**: `Amazon Redshift` (default) or `Amazon Glue`. Controls which asset type is pulled from the SageMaker Search API and determines the `DataSourceType` attribute on ingested Database assets. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `systemDomain` (required) — domain where System assets are created. +- `community` (required) — community under which per-database and per-schema sub-domains are auto-created. +- `datasource` (required) — `Amazon Redshift` or `Amazon Glue`. +- `regions` (required, array) — one or more AWS regions to scan; at least one must be provided. +- `datazoneDomains` (optional, array) — SageMaker Unified Studio domain IDs to include; empty means all domains visible to the connection. +- `jdbcConnectionsMapping` (optional, Redshift only) — array of `{ databaseName, connectionId }` pairs that link a Collibra Database full name to a JDBC connection, enabling profiling, classification, and sampling on that database. + +## Gotchas +- The IAM user must have `datazone:ListDomains`, `datazone:ListProjects`, and `datazone:SearchListings` permissions in every region you scan. +- `jdbcConnectionsMapping` is only relevant for Redshift. Glue tables are S3-backed; JDBC profiling is not supported in v1, so the field is hidden in the UI when `Amazon Glue` is selected. +- For Glue, all Glue databases within a catalog are mapped as schemas under a single synthetic `AwsDataCatalog` Database — there is no real per-catalog database concept in DataZone listings. +- The capability-level parameter `jdbc-capabilities-auto-create-mode` (set on `edge_create_capability`, not in the generic config) controls whether JDBC ingestion, profiling, and classification capabilities are created automatically when the integration runs. Defaults to auto-creating ingestion + profiling + classification. diff --git a/pkg/skills/files/collibra/etl-integration/references/fabric.md b/pkg/skills/files/collibra/etl-integration/references/fabric.md new file mode 100644 index 0000000..c8d14ec --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/fabric.md @@ -0,0 +1,57 @@ +# Microsoft Fabric (`fabric-synchronization`) + +Extracts metadata from Microsoft Fabric — traversing Tenant → Capacity → Workspace → +Lakehouses (schemas, tables, columns, files), Warehouses (schemas, tables, views, columns), +SQL Databases, and Mirrored Azure Databricks Catalogs — and ingests it into DGC. +`capability-type` (the `typeId` for `edge_create_capability`): **`fabric-synchronization`**. + +**Docs:** [About the Microsoft Fabric integration](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/MicrosoftFabric/co_about-fabric.htm) + +**Connection type:** Azure. Auth method (set on `edge_create_connection`): +- **Service Principal** — `client-id`, `client-secret`, and `tenant-id` of an Azure Entra + App registration (OAuth 2.0 client credentials flow). + +**Workflows:** inbound is `main` (default — omit `workflow`). No outbound flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `community` (required) — Collibra community where per-workspace Technology Assets + subdomains are auto-created. +- `mainDomain` (required) — domain where `MicrosoftTenant` and `FabricCapacity` assets land. +- `workspaceNames` (optional) — limit ingestion to specific workspace names; empty = all + accessible to the service principal. +- `maxLakehouseFiles` (optional, default `100`) — cap on files ingested per Lakehouse; set + to `-1` to ingest all files. +- `jdbcConnectionsMapping` (optional) — pairs `databaseName` (full Collibra asset name, e.g. + `Tenant>Capacity>Workspace>my-db`) with a `connectionId` (JDBC connection) to unlock + schema/table/column ingestion, profiling, classification, and sampling for SQL Databases. + Without this, the capability creates only the top-level `Database` asset with no internal + structure. +- `domainIncludeMappings` / `domainExcludeMappings` — route or exclude assets by a + `>`-separated hierarchical path (1–4 segments: Workspace → Artifact → Schema → Table; + wildcards `*` and `?` supported within each segment). Excludes always win over includes. +- `tenantNameOverride` / `capacityNameOverrides` — static name overrides; once set, the + capability stops updating those names from the Microsoft API. + +## Gotchas +- **SQL Database internal structure is not ingested by this capability.** Only the top-level + `Database` asset is created. Configure a `jdbcConnectionsMapping` entry so the + auto-created JDBC Ingestion capability handles schemas, tables, and columns. +- **Warehouse metadata requires `ReadData` permission**, not just Workspace Viewer. The Viewer + role grants only `CONNECT`; `ReadData` (*Read all data using SQL*) must be explicitly granted + in the Fabric portal to allow JDBC `INFORMATION_SCHEMA` queries. +- **Lakehouse table and file access is governed by OneLake security roles**, separate from the + workspace Viewer role. Grant the Service Principal **Read on Tables** (schema/column + discovery via OneLake Table API) and **Read on Files** (file listing via OneLake DFS API) in + the Lakehouse's OneLake security settings — or grant item-level `ReadAll` to bypass role + checks entirely. +- **Mirrored Azure Databricks Catalog tables require two Databricks-side controls**: (1) + `EXTERNAL USE SCHEMA` privilege on each schema in Unity Catalog, and (2) **External data + access enabled on the metastore** (a metastore-level toggle, off by default, requiring + metastore/account admin). If either control is missing, table listing silently returns HTTP + 200 with no tables and column fetches return 403 — ingestion appears to succeed but produces + empty schemas. +- **No admin-consent Power BI/Fabric application permissions** may be registered in Entra for + the Service Principal, even unused ones. Their mere presence breaks service principal + authentication against the Power BI Admin API (used for capacity discovery). +- When both `workspaceNames` and `domainIncludeMappings` are configured, a workspace must + pass **both** filters to be ingested. diff --git a/pkg/skills/files/collibra/etl-integration/references/index.md b/pkg/skills/files/collibra/etl-integration/references/index.md new file mode 100644 index 0000000..fa1d336 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/index.md @@ -0,0 +1,40 @@ +# ETL integration catalog + +Every integration below is an Edge capability configured through the same lifecycle (see +the parent `SKILL.md`): `edge_create_connection` → `edge_create_capability` → +`catalog_etl_get_schema` → `catalog_etl_save_generic_config` → `catalog_etl_start_job` / +`catalog_etl_add_schedule`. + +The **capability typeId** is what you pass to `edge_create_capability` (`typeId`). Always +confirm the exact connection/capability parameters via `edge_list_capability_types` and the +generic-config fields via `catalog_etl_get_schema` — the per-integration notes here are +conceptual, not field-level. + +| Integration | capability typeId | Syncs | +|---|---|---| +| Google Knowledge Catalog / Dataplex | `dataplex-synchronization` | GCP Dataplex / Knowledge Catalog metadata (BigQuery, CloudSQL) — see `dataplex.md` | +| Google Vertex AI | `vertex-synchronization` | Vertex AI models/datasets (GCP) | +| GCP technical lineage | `dataplex-lineage-synchronization` | Technical lineage for GCP | +| Databricks Unity Catalog | `databricks-edge-capability` | Databricks Unity Catalog metadata | +| Databricks lineage | `databricks-lineage` | Technical lineage for Databricks Unity Catalog | +| AWS Bedrock | `bedrock-synchronization` | AWS Bedrock AI models | +| AWS SageMaker | `sagemaker-synchronization` | SageMaker AI (models, etc.) | +| AWS DataZone / SageMaker Unified Studio | `datazone-synchronization` | SageMaker Unified Studio data catalog (Preview) | +| Azure ML | `azureml-synchronization` | Azure Machine Learning | +| Azure AI Foundry | `azure-foundry-synchronization` | Azure AI Foundry | +| Microsoft Fabric | `fabric-synchronization` | Microsoft Fabric | +| Microsoft Purview | `purview-synchronization` | Purview catalog metadata | +| MLflow | `mlflow-synchronization` | MLflow AI experiments/models | +| Sigma | `sigma-synchronization` | Sigma dashboards + lineage | +| ThoughtSpot | `thoughtspot-synchronization` | ThoughtSpot objects + lineage (Preview) | + +**Not listed here / not controllable via chip:** the object-storage crawlers **S3** +(`s3-synchronization`), **ADLS** (`adls-synchronization`), and **GCS** +(`gcs-synchronization`) use a different endpoint/config model and cannot be configured +through the `catalog_etl_*` tools — see "Not controllable via chip (yet)" in `SKILL.md`. + +Notes: +- ThoughtSpot's manifest uses a generated typeId; confirm the live value with + `edge_list_capability_types` if `edge_create_capability` rejects the id above. +- SAP capabilities (AI Core, Datasphere, Data Products, S/4HANA) are out of scope for this + skill — they're owned by the JDBC Ingestion team. diff --git a/pkg/skills/files/collibra/etl-integration/references/mlflow.md b/pkg/skills/files/collibra/etl-integration/references/mlflow.md new file mode 100644 index 0000000..63c4b31 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/mlflow.md @@ -0,0 +1,24 @@ +# MLflow AI (`mlflow-synchronization`) + +Extracts metadata from an MLflow model registry and ingests `MlflowAIModel` assets into a +Collibra Technology Asset Domain. `MlflowAIModel` is a subtype of `AIModel` in the Collibra +AI Governance operating model; the capability creates one asset per registered model (latest +READY version only). + +**Docs:** [About the MLflow AI integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/MLfLowAI/co_about-mlflow.htm) + +**Connection type:** `generic` (HTTP-based). Auth methods (set on `edge_create_connection`): +- **Basic Authentication** — `username` and `password` credentials against the MLflow tracking server URL. +- **OAuth (Client Credentials)** — `tokenUrl`, `clientId`, and `clientSecret`; compatible with Databricks OAuth (M2M). + +**Workflows:** single inbound workflow `main` (default — omit `workflow`). No outbound flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — UUID of the Technology Asset Domain where `MlflowAIModel` assets are created or updated. +- `customAiMetricsMappings` (optional) — array of `{ from, to }` objects mapping an MLflow metric key (e.g., `f1_score`) to a Collibra attribute type UUID. Extends the four standard auto-mapped metrics (`accuracy`, `precision_score`, `mae`, `mse`). A custom mapping entry can also override a standard metric, storing the value in both the standard attribute and the custom one. + +## Gotchas +- Only the **latest READY version** of each registered model is ingested. Models with no `READY` versions are silently skipped — no asset is created or updated. +- If a model version has no associated `run_id`, or the training run was deleted (MLflow returns 404 on `/runs/get`), all metric attributes are silently omitted for that asset. +- Every attribute type referenced in `customAiMetricsMappings` must already be **assigned to the `MlflowAIModel` asset type** (`00000000-0000-0000-0000-000000031410`) in Collibra before the sync runs. Validation happens at startup; a missing assignment aborts the entire synchronization. +- The four standard metric keys (`accuracy`, `precision_score`, `mae`, `mse`) are hard-coded. If the MLflow run uses different key names (e.g., `acc` instead of `accuracy`), the standard attributes will be empty — use `customAiMetricsMappings` to remap them. diff --git a/pkg/skills/files/collibra/etl-integration/references/purview.md b/pkg/skills/files/collibra/etl-integration/references/purview.md new file mode 100644 index 0000000..1bc8fd7 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/purview.md @@ -0,0 +1,37 @@ +# Microsoft Purview (`purview-synchronization`) + +Extracts schema metadata from Azure SQL Server databases cataloged in Microsoft Purview (servers, +databases, schemas, tables, and columns) and ingests it into DGC. `capability-type` (the `typeId` +for `edge_create_capability`): **`purview-synchronization`**. + +**Docs:** No standalone Collibra product page documents this `purview-synchronization` (Azure SQL Server via Purview) capability. In the public docs, "Purview" appears only as a *synchronization source within the ADLS integration* — a different capability — so do not treat the ADLS docs as authoritative for this integration. + +**Connection type:** Azure. Auth method (set on `edge_create_connection`): +- **Service Principal** — `clientId`, `clientSecret`, and `tenantId` from an Azure AD app + registration. + +**Workflows:** single inbound workflow `main` (default — omit `workflow`). No outbound flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `dataSource` (required) — currently only `MSSQL` (Azure SQL Server); defaults to `MSSQL`. +- `system` (required) — target System asset in DGC to which databases are linked. +- `community` (required) — default Collibra community; sub-domains for databases and schemas are + auto-created here when no explicit domain mapping matches. +- `domainIncludeMappings` (optional) — array of `{from, to}` objects. `from` is a + `Server > Database > Schema` glob path (supports `*` and `?` wildcards); `to` is the target + domain. Omitting `to` auto-creates domains inside the default community. +- `domainExcludeMappings` (optional) — array of strings in `Server > Database > Schema` format; + matched paths are skipped even if they match an include rule. + +## Gotchas +- This integration reads from Purview's catalog — it does not scan Azure SQL directly. The Azure + SQL data sources must have already been registered and scanned inside Purview before running the + capability, otherwise no assets will be found. +- The service principal used in the Azure connection needs **Data Reader** and **Data Source Admin** + roles assigned in the Purview account (set in Purview's data-map collections permissions, not in + Azure IAM alone). +- The `purview-account` capability parameter (the short Purview account name, not a URL) is set at + capability creation time and is separate from the generic config. +- Domain mapping paths must be a triplet `Server > Database > Schema`. Paths with fewer or more + levels fail validation at run time with a descriptive error. +- Exclude mappings take priority over include mappings. diff --git a/pkg/skills/files/collibra/etl-integration/references/sagemaker.md b/pkg/skills/files/collibra/etl-integration/references/sagemaker.md new file mode 100644 index 0000000..7982075 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/sagemaker.md @@ -0,0 +1,26 @@ +# AWS SageMaker AI (`sagemaker-synchronization`) + +Extracts ML model metadata from Amazon SageMaker AI and ingests it into DGC as a structured AI asset hierarchy: AI Base Models → AI Model Versions → AI Model Deployments → AI Endpoints + AI Monitors. The `typeId` for `edge_create_capability` is **`sagemaker-synchronization`**. + +**Docs:** [About the AWS SageMaker AI integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/AWSSageMaker/co_about-aws-sagemaker-ai.htm) + +**Connection type:** AWS. Auth methods (set on `edge_create_connection`): +- **IAM** — programmatic user; requires `aws_access_key_id` and `aws_secret_access_key`. +- **EC2** — instance role; no explicit credentials needed (suitable when the Edge site runs on an EC2 instance with an attached IAM role). + +**Workflows:** single inbound workflow `main` (omit the `workflow` argument). No outbound flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — target Collibra Domain where all AI assets are created (AI Base Models, AI Model Versions, AI Model Deployments, AI Endpoints, AI Monitors). +- `regions` (optional) — list of AWS region identifiers to scope ingestion; omit to scan all SageMaker-enabled regions automatically. +- `ingestDeploymentOutput` (optional, default `false`) — when enabled, endpoint output paths from each endpoint configuration are ingested as `FileContainer` and `File` assets linked to the corresponding `AIModelDeployment`. +- `customAiMetricsMappings` (optional) — maps SageMaker metric key names (supports nested JSON lookup) to DGC attribute type UUIDs on the AI model asset. Useful for ingesting non-standard or user-defined model metrics beyond the built-in accuracy/precision/MSE/MAE fields. + +## Gotchas +- Only model packages with `ModelApprovalStatus = Approved` are ingested. Non-approved versions are silently skipped — if model versions are missing, check their approval status in SageMaker. +- The ETL path is selected automatically at runtime based on DGC version: **≥ 2026.03** produces the full AI asset hierarchy (Base Model → Version → Deployment → Endpoint + Monitor); **< 2026.03** produces a single legacy `AWSSagemakerAIModel` asset per model package group (latest approved version only). No configuration is required or possible. +- The IAM user (or role) needs separate permission sets for model ingestion (`sagemaker:ListModelPackageGroups`, `DescribeModelPackageGroup`, `ListModelPackages`, `DescribeModelPackage`), endpoint/deployment ingestion (`sagemaker:ListEndpoints`, `DescribeEndpoint`, `DescribeEndpointConfig`, `DescribeModel`), and monitoring ingestion (`sagemaker:ListMonitoringSchedules`, `DescribeMonitoringSchedule`, `ListMonitoringAlerts`). Missing any group silently drops that category of assets. +- If a deployment variant cannot be linked to an AI Model Version (by package ARN or model data URL), it is still ingested as a standalone deployment with no version relation. The sync report records the failure, but no error is raised. +- An AI Endpoint asset is only created if at least one of its deployment variants was successfully resolved; endpoints with zero resolved variants are dropped. +- `customAiMetricsMappings.from` performs a recursive key search across the metrics JSON. If the found value is a complex object containing a `"value"` key, that inner value is extracted; otherwise the entire object is serialized to a string. Map leaf metric keys (e.g., `recall`, `accuracy`) rather than parent objects to get clean scalar values. +- The `AIMonitor.Url` attribute is only populated on DGC ≥ 2026.04. The `InitiatingUserInSource` attribute on AI assets requires DGC ≥ 2026.07. diff --git a/pkg/skills/files/collibra/etl-integration/references/sigma.md b/pkg/skills/files/collibra/etl-integration/references/sigma.md new file mode 100644 index 0000000..f21d736 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/sigma.md @@ -0,0 +1,23 @@ +# Sigma Computing (`sigma-synchronization`) + +Extracts metadata from Sigma Computing (workspaces, workbooks, datasets, data models, connections, and their columns/elements) and ingests it into DGC. `capability-type` (the `typeId` for `edge_create_capability`): **`sigma-synchronization`**. + +**Docs:** [About Sigma integration](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/Sigma/co_about-sigma-integration.htm) + +**Connection type:** Sigma (OAuth 2.0 client credentials). Auth parameters (set on `edge_create_connection`): +- `clientId` — OAuth client ID from Sigma Admin Portal → Developer Access. +- `clientSecret` — OAuth client secret (shown only once at creation; store it safely). +- `url` — Region-specific Sigma REST API base URL (e.g. `https://aws-api.sigmacomputing.com`; also `gcp-api`, `azure-api` variants). +- `tenantUrl` — Your Sigma tenant URL (e.g. `https://app.sigmacomputing.com/`); used to build asset `Url` attributes. + +**Workflows:** inbound is `main` (default — omit `workflow`). No outbound (DGC → Sigma) flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — UUID of the DGC domain where all ingested assets will land. +- `workspaces` (optional) — array of workspace IDs or case-insensitive workspace names to limit scope. Empty = all workspaces the OAuth client can see. The catalog UI pre-populates this field as a dropdown via an automatic lightweight pre-flight call. + +## Gotchas +- The OAuth client must have **Admin** or **Creator** account type in Sigma. A Member-level client cannot enumerate org-wide content reliably. +- Admin account type alone lets the capability list a workspace and create the workspace asset, but **not** walk its contents (folders, workbooks, datasets, data models). Those workspaces appear in the sync report as "could not be synchronized." Fix: add the OAuth client as a workspace member with at least Viewer permissions. +- Setting `workspace-ids` on the capability **and** `workspaces` in the catalog config simultaneously throws `InvalidParametersException` at startup. Use only one. +- Workspace filter entries that match neither a workspace ID nor a workspace name are tracked as failures. If all configured entries fail, the job status becomes `FAILED`; partial failures yield `COMPLETED_WITH_ERRORS`. diff --git a/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md b/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md new file mode 100644 index 0000000..a7c4209 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md @@ -0,0 +1,37 @@ +# ThoughtSpot Synchronization (`thoughtspot-synchronization`) + +Extracts metadata from a ThoughtSpot cloud tenant — Tenant, Orgs, Connections, Liveboards, Answers, Answer Visualizations, Data Models (Worksheets), Data Entities (Tables/Views), and Data Attributes (columns) — and ingests it into a Collibra DGC domain. A separate lineage workflow additionally submits column-level lineage edges to Collibra TechLin. The `capability-type` (the `typeId` for `edge_create_capability`) is **`thoughtspot-synchronization`** (confirmed in `manifest-template.yaml`; the published `manifest.yaml` uses the `` build-time placeholder, so the resolved value is the same string). + +**Docs:** [About ThoughtSpot integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/ThoughtSpot/co_about-thoughtspot-integration.htm) + +**Connection type:** Generic. Auth method (set on `edge_create_connection`): +- **Username / Password** — ThoughtSpot username (`userName`), password (`password`), and tenant base URL (`url`, e.g. `https://customer.thoughtspot.cloud`). The capability authenticates via the ThoughtSpot REST API v2 full-token endpoint and re-authenticates per org to obtain an org-scoped bearer token. + +**Workflows:** The integration exposes two runnable tabs in the catalog Integration Configuration page: +- **Metadata Inbound** (`default` workflow, `-DstartMode=sync`) — inbound; fetches all ThoughtSpot entity types listed above and creates Catalog assets in the configured DGC domain. +- **ThoughtSpot Lineage** (`lineage` workflow, `-DstartMode=lineage`, Preview) — submits column-level and asset-level lineage relations to Collibra TechLin. Requires the **Metadata Inbound** tab to be configured first. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) + +**Metadata Inbound tab:** +- `domain` (required) — UUID of the DGC domain where ThoughtSpot assets will be created. +- `orgs` (optional) — multi-select list of ThoughtSpot org IDs or names to include. Leave empty to sync all orgs the credentials can see. + +**ThoughtSpot Lineage tab:** +- `domain` (required) — UUID of the DGC domain that owns the lineage source. +- `sourceId` (required) — stable string identifier used to construct the lin-sdk source name (`ThoughtSpot--`). Changing this across runs creates a new parallel lineage source instead of updating the existing one. +- `orgs` (optional) — same org-filter semantics as the sync tab; should be aligned with the sync configuration to avoid partial graphs. + +The lineage capability also requires these **install-time** run-parameters (not in the generic config): `targetDgc` (DGC URL), `username` and `password` (Collibra DGC credentials for lin-sdk — distinct from the ThoughtSpot connection credentials). Optionally `customParameters.techlinHost` + `customParameters.techlinKey` if bypassing DGC-routed lineage submission and pushing to the Techlin server directly. + +## Gotchas + +- **Run sync before lineage.** Lineage edges stitch onto assets the sync flow has already created. Running lineage against an empty domain produces a disconnected gray graph in TechLin with no errors. +- **Do not set `org-ids` at the capability level and `orgs` in the generic config at the same time** — the capability throws `InvalidParametersException` at startup. Use one or the other. +- **`sourceId` must be stable.** Changing it between lineage runs creates a new lin-sdk source instead of overwriting the previous one, resulting in duplicate lineage graphs. +- **Lineage is all-or-nothing per run.** If any org's metadata fetch fails during the lineage workflow, the entire job aborts before submitting any edges to TechLin. The sync workflow does not share this property. +- **Data Entities are silently dropped** if the capability cannot resolve their ThoughtSpot Connection (by `dataSourceId` or by name). Their columns are dropped too. This happens when the Connection is not visible to the service user or when multiple connections share a name and the fallback name-match is ambiguous. +- **Private / embed ThoughtSpot deployments** (non-`*.thoughtspot.cloud` hosts) get an incorrect `Url` attribute on the Tenant asset — the synchronization itself still works because API calls use the user-supplied connection URL. +- **Only TABLE-type visualizations** produce per-column Data Attribute assets. CHART, HEADLINE, and other viz types are not expanded into attributes, and the lineage flow cannot draw column-level edges into them. +- **The `defaultStatus` and `cloudIngestionJobId` run-parameters** are declared in the manifest but are not read by the current code — setting them has no effect. +- **`save-input-metadata=true`** (install-time flag) records all source metadata in a downloadable DGC file. Use only when requested by Collibra Support — the file can be large and contains source-system content. diff --git a/pkg/skills/files/collibra/etl-integration/references/vertex.md b/pkg/skills/files/collibra/etl-integration/references/vertex.md new file mode 100644 index 0000000..a5779f0 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/vertex.md @@ -0,0 +1,38 @@ +# Google Vertex AI (`vertex-synchronization`) + +Extracts AI/ML model metadata from Google Vertex AI (Model Registry, endpoints, +deployments, monitors, reasoning-engine agents, and referenced Model Garden +foundation models) and ingests it into DGC. `capability-type` (the `typeId` for +`edge_create_capability`): **`vertex-synchronization`**. + +**Docs:** [About the Google Vertex AI integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/GoogleVertexAI/co_about-vertex-ai.htm) + +**Connection type:** GCP. Auth methods (set on `edge_create_connection`): +- **Service Account** — full JSON key (`gcp_service_account_credentials_json`). +- **Workload Identity Federation (WIF)** — federated identity; token/URL for file-based WIF. +- **WIF using GKE** — when the Edge site runs in GKE; uses the site's federated identity, credentials left blank. + +**Workflows:** inbound only, `main` (default — omit `workflow`). There is no outbound +(DGC → Vertex) flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — target domain for the ingested assets. +- `projects` — GCP Project IDs to scan; if empty, auto-discovers every project the + service account can access. +- `locations` — regions to search; if empty, searches all locations where Vertex AI is available. +- `ingestAiInputAndOutput` (default true) — also ingest training-data File/File Container + assets from GCS and BigQuery prediction-logging output tables; when false, only model + metadata is ingested. +- `systems` — Collibra System assets to search for existing BigQuery output tables before + creating new ones (dedup); only applies when `ingestAiInputAndOutput` is enabled. +- `customLabelMappings` / `customAiMetricsMappings` — map custom Vertex labels/metrics to + AI Model attribute types. + +## Gotchas +- Vendor foundation models (Gemini, Claude, Llama, Mistral, etc.) are ingested **on demand + only** when a customer model references one via `baseModelSource.modelGardenSource`; the + Model Garden catalogue is never proactively scanned. +- If `getPublisherModel` fails (permission/not-found/unavailable), the vendor parent asset + and its parent-link relation are skipped but the sync continues. +- GCS File assets and BigQuery output tables are only produced when `ingestAiInputAndOutput` + is enabled — extra permissions (`storage.objects.*`, `bigquery.tables.get`) are then required. diff --git a/pkg/skills/files/collibra/index/SKILL.md b/pkg/skills/files/collibra/index/SKILL.md index 46337e8..c5719b3 100644 --- a/pkg/skills/files/collibra/index/SKILL.md +++ b/pkg/skills/files/collibra/index/SKILL.md @@ -1,6 +1,6 @@ --- description: Navigator for chip's Collibra skills. Start here when unsure which skill applies. -related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context +related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/jdbc-ingestion, collibra/etl-integration --- # Collibra skills — navigator @@ -19,6 +19,8 @@ must be bridged to another, and which permissions are required. | Modify an existing asset's attributes, relations, tags, status, or owners | `collibra/asset-edit` | | Register a table (and its dimension tables) as a Collibra Data Product with ports | `collibra/data-product-create` | | Generate governed YAML context (semantic blueprints, metric definitions) for an asset | `collibra/context` | +| Set up an Edge JDBC connection/capability and run jdbc-ingestion | `collibra/jdbc-ingestion` | +| Set up any metadata-sync ETL integration (Dataplex, Databricks UC, Purview, Sigma, …) — connection, capability, generic config, schedule/run | `collibra/etl-integration` | If a task is a single tool call with no chaining (e.g. `get_asset_details` by UUID, `list_asset_types`, `pull_data_contract_manifest`), no skill is needed — the tool's own diff --git a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md new file mode 100644 index 0000000..ca5e552 --- /dev/null +++ b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md @@ -0,0 +1,156 @@ +--- +description: Set up an Edge JDBC connection and capability, register a database, and run jdbc-ingestion end to end. +related: collibra/discovery +--- + +# JDBC ingestion — connect, register, and run + +Setting up jdbc-ingestion is a fixed sequence across two separate systems: Edge +(connection + capability) and DGC catalog (database registration + the actual sync). +No single tool does the whole thing — skipping a step or calling tools out of order is +the most common way this goes wrong. + +## First: confirm this is the right ingestion path + +Many data sources support **more than one** ingestion path — most commonly a native +**ETL integration** (`collibra/etl-integration`, e.g. Databricks Unity Catalog, Dataplex, +Purview, Sigma) *and* generic **JDBC** (this skill). The native path is usually richer +(lineage, finer metadata) and preferred where it exists. + +**Whenever the user asks to create an integration, before doing anything else, determine +which ingestion type it should be** — don't assume JDBC just because this skill is loaded: + +1. Identify the data source. +2. Check whether more than one path supports it (consult `collibra/etl-integration`'s + `references/index.md` for the native-ETL data sources). +3. **If more than one path is possible, ask the user which they want** before starting. + Only proceed with this JDBC skill once the user has chosen JDBC (or JDBC is the only + supported path). + +## The full sequence + +1. **`edge_list_sites`** — get the `edgeSiteId` you'll use everywhere below. +2. **`get_data_source_setup_guide`** (pass the data source name, e.g. `"Snowflake"`) — + before guessing any connection parameter. Returns the driver class, connection + string format, the Marketplace URL for the driver jar, and per-auth-method + properties. Treat a failed/empty lookup as "ask the user for the connection + properties directly," not as a bug to retry. +3. **`upload_file`** — upload the JDBC driver jar. **Only use a file the user actually + provides, sourced from the Collibra Marketplace listing from step 2** — never fetch + a driver yourself from anywhere else, and never invent one. If the file is too large + for `contentBase64` (most JDBC drivers are tens of MB) and this chip instance + doesn't have `filePath` enabled, stop and guide the user to create the connection + manually in the Edge UI instead — see "Large driver files" below. +4. **`edge_create_connection`** — `typeId: "Generic"` (not a vendor-specific connection + type — jdbc-ingestion requires the Generic JDBC connection type). Fixed manifest + parameters (`driver-class`, `connection-string`, `driver-jar` as the `upload_file` + URI) go in `parameters`; open-ended vendor properties (Snowflake's Role, Warehouse, + User, private key file, etc.) go in `additionalProperties`, not `parameters`. +5. **`test_connection`** — verify it actually connects before building on top of it. + Pass `timeoutSec` for a synchronous result, or poll the returned `jobId` with + `edge_get_job_status`. +6. **`edge_create_capability`** — `typeId: "jdbc-ingestion"`, `parameters.connection` set to + the connection id from step 4. +7. Catalog scaffolding, only if it doesn't already exist: **`create_community`** → + **`find_domain_types`** (look up e.g. "Physical Data Dictionary") → + **`create_domain`**. **`find_users`** to resolve owner names to UUIDs for the next + step. +8. **`register_database`** — discovers the database through the Edge connection and + registers it as a Database asset. Needs `communityId`, `parentSystemId` (an existing + System asset), and `ownerIds` (from step 7). **Never default to registering + whichever database is discovered first if more than one exists — always confirm + which one with the user.** The tool refuses to guess when there's more than one + database (see "Confirm scope" below). If it returns a retryable-looking error, it + usually means the underlying async refresh hasn't completed yet (cold-start data + source connections can take 30–60s) — just call it again; it's idempotent. +9. **`configure_database_schemas`** — takes the `databaseConnectionId` from step 8's output, + discovers its schemas, and sets which schemas/tables get synchronized. **Never + default to "sync everything" — always confirm which schemas and which tables with + the user first.** The tool refuses to guess when there's more than one schema. This + is a separate call from step 8 specifically so that a schema-ambiguity error here + never risks colliding with the database step 8 already registered — safe to call + again later to change what's synced. +10. **`start_ingestion`** — triggers the actual sync using the database id from step 8. +11. **`get_job_status`** — poll the job id `start_ingestion` returned. A + 202/success from `start_ingestion` only means the job was accepted, not that + ingestion finished — always poll to confirm completion. + +## Two job-id spaces — do not cross them + +- **Edge-site jobs** (from `test_connection`) → poll with **`edge_get_job_status`**. +- **DGC jobs** (from `start_ingestion`) → poll with **`get_job_status`**. + +Passing one tool's job id to the other will fail or silently poll the wrong resource. +`start_ingestion`'s own `Job.id` is a catalog job id — never poll it with +`edge_get_job_status`. + +## Do not use a raw "run capability" tool for this + +If this chip instance also exposes a generic "run capability" tool (e.g. +`edge_run_capability`, which wraps Edge's `POST /capabilities/{id}/run` directly): +**never use it to trigger a jdbc-ingestion run.** Always use `start_ingestion` instead. +`start_ingestion` goes through DGC's `synchronizeMetadata` endpoint, which does the +catalog database-registration bookkeeping (linking the job back to the Database asset, +updating sync state) that a raw capability run skips entirely — a direct run would +appear to succeed on the Edge side while leaving the catalog side out of sync. + +## Confirm scope before registering and configuring the database + +`register_database` and `configure_database_schemas` will not silently register the wrong +database or sync every table in every schema — both refuse ambiguity rather than guess, +and neither has a side effect until its own ambiguity is resolved: + +- **`register_database`'s `databaseName`** is required if the data source exposes more + than one database/catalog through the connection. The tool errors and names the + discovered candidates if you omit it while more than one exists — no Database asset + is registered until this is resolved. +- **`configure_database_schemas`'s `schemaNames`** is required if the database has more than + one schema. Same behavior: omit it with multiple schemas discovered and the tool + errors, listing them. Pass the literal `["*"]` to configure every schema — only do + this after the user has actually said they want everything, not as a shortcut when + you don't know yet. +- **`configure_database_schemas`'s `include`** has no default. You must always pass a table + pattern (`"*"` for all tables, or a specific comma-separated pattern) — there is no + silent "sync everything" fallback. + +The practical flow for each tool: call it once with just the required identifiers and +let it fail on ambiguity to discover the candidates, **or** if you already know +multiple exist, ask the user up front before calling at all. Either way, do not pass +`"*"` for `schemaNames` or `include` on your own initiative — confirm with the user +first. The one exception: if discovery finds exactly one database (or one schema), the +respective tool auto-selects it — no ambiguity to resolve, so no need to ask about +*which* one, though you should still confirm the table pattern (`include`/`exclude`). + +## Picking up an existing connection instead of duplicating one + +If a connection with the driver already exists (most commonly because the driver file +was too large to upload and the user created it manually via the Edge UI), use +**`edge_find_connections`** (by name, optionally scoped to `edgeSiteId`) to get its id and +skip straight to step 6 — do not call `edge_create_connection` again for it. + +## Large driver files + +`upload_file` takes file content as base64 in a tool call, which has a hard ceiling +that isn't about encoding — the content has to be generated by the calling model +itself, so a large jar (tens of MB) simply won't fit. Some chip instances (the +standalone `chip` binary, not chip-service) also accept a `filePath` input to read +the file directly off local disk instead. If neither `filePath` nor a small-enough +file is available: guide the user to create the whole connection manually via the Edge +UI (Settings > Edge > site > Connections > New), giving them the exact field values +(driver class, connection string — from `get_data_source_setup_guide`) so they only +need to pick the file and click through. Once they confirm it's saved, use +`edge_find_connections` to pick it up and continue from step 6. + +## Hard rules + +1. **Never source a JDBC driver from anywhere but Collibra Marketplace.** If in doubt, + ask the user to download the jar from the URL `get_data_source_setup_guide` returns + and hand you the file — don't fetch or substitute one yourself. +2. **`edge_create_connection`'s `typeId` for jdbc-ingestion is `"Generic"`,** never a + vendor-specific connection type — jdbc-ingestion cannot use those. +3. **Always poll after `start_ingestion`.** A successful call only means the job was + accepted. +4. **Never pick a database, schema set, or table scope for the user.** When more than + one is possible, ask; when `register_database` or `configure_database_schemas` reports + ambiguity, relay the candidates and ask rather than picking one yourself or + retrying with `"*"`. diff --git a/pkg/tools/catalog_etl_add_schedule/tool.go b/pkg/tools/catalog_etl_add_schedule/tool.go new file mode 100644 index 0000000..d52d600 --- /dev/null +++ b/pkg/tools/catalog_etl_add_schedule/tool.go @@ -0,0 +1,53 @@ +package catalog_etl_add_schedule + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` + CronExpression string `json:"cronExpression" jsonschema:"the cron expression for the schedule (e.g. '0 2 * * *')"` + CronTimeZone string `json:"cronTimeZone" jsonschema:"the time zone for the cron schedule (e.g. 'UTC', 'America/New_York')"` + CronJson string `json:"cronJson,omitempty" jsonschema:"optional JSON representation of the cron schedule"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` +} + +type Output struct { + Schedule *clients.GenericSchedule `json:"schedule,omitempty" jsonschema:"the created schedule"` + Success bool `json:"success" jsonschema:"whether the schedule was added successfully"` + Error string `json:"error,omitempty" jsonschema:"error message if adding the schedule failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_add_schedule", + Description: "Adds a synchronization schedule to an ETL integration. Requires a cron expression and time zone.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schedule, err := clients.AddGenericSchedule(ctx, collibraClient, input.IngestibleId, input.Workflow, clients.AddGenericScheduleRequest{ + CronExpression: input.CronExpression, + CronTimeZone: input.CronTimeZone, + CronJson: input.CronJson, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to add schedule: %s", err.Error())}, nil + } + return Output{Schedule: schedule, Success: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_add_schedule/tool_test.go b/pkg/tools/catalog_etl_add_schedule/tool_test.go new file mode 100644 index 0000000..e41abe8 --- /dev/null +++ b/pkg/tools/catalog_etl_add_schedule/tool_test.go @@ -0,0 +1,72 @@ +package catalog_etl_add_schedule_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_add_schedule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericAddSchedule(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddGenericScheduleRequest) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{Id: 1, CronExpression: in.CronExpression, CronTimeZone: in.CronTimeZone} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } + if output.Schedule == nil || output.Schedule.CronExpression != "0 2 * * *" { + t.Fatal("expected cron expression 0 2 * * *") + } +} + +func TestCatalogGenericAddScheduleAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusBadRequest, "bad request" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericAddScheduleInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "bad", + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + }) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_cancel_job/tool.go b/pkg/tools/catalog_etl_cancel_job/tool.go new file mode 100644 index 0000000..033ae58 --- /dev/null +++ b/pkg/tools/catalog_etl_cancel_job/tool.go @@ -0,0 +1,44 @@ +package catalog_etl_cancel_job + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the job was cancelled"` + Error string `json:"error,omitempty" jsonschema:"error message if cancellation failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_cancel_job", + Description: "Cancels the currently running synchronization job for an ETL integration.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + if err := clients.CancelGenericJob(ctx, collibraClient, input.IngestibleId, input.Workflow); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to cancel job: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_cancel_job/tool_test.go b/pkg/tools/catalog_etl_cancel_job/tool_test.go new file mode 100644 index 0000000..2b95c95 --- /dev/null +++ b/pkg/tools/catalog_etl_cancel_job/tool_test.go @@ -0,0 +1,60 @@ +package catalog_etl_cancel_job_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/catalog_etl_cancel_job" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericCancelJob(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/cancel", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusNoContent, nil + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestCatalogGenericCancelJobAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/cancel", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericCancelJobInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_delete_generic_config/tool.go b/pkg/tools/catalog_etl_delete_generic_config/tool.go new file mode 100644 index 0000000..801d134 --- /dev/null +++ b/pkg/tools/catalog_etl_delete_generic_config/tool.go @@ -0,0 +1,43 @@ +package catalog_etl_delete_generic_config + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration whose configuration to delete"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the configuration was deleted"` + Error string `json:"error,omitempty" jsonschema:"error message if deletion failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_delete_generic_config", + Description: "Deletes the configuration for an ETL integration.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + if err := clients.DeleteGenericConfig(ctx, collibraClient, input.IngestibleId); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to delete configuration: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_delete_generic_config/tool_test.go b/pkg/tools/catalog_etl_delete_generic_config/tool_test.go new file mode 100644 index 0000000..cf478a5 --- /dev/null +++ b/pkg/tools/catalog_etl_delete_generic_config/tool_test.go @@ -0,0 +1,60 @@ +package catalog_etl_delete_generic_config_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/catalog_etl_delete_generic_config" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericDeleteConfig(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusNoContent, nil + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestCatalogGenericDeleteConfigAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericDeleteConfigInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_delete_schedule/tool.go b/pkg/tools/catalog_etl_delete_schedule/tool.go new file mode 100644 index 0000000..7b2ced6 --- /dev/null +++ b/pkg/tools/catalog_etl_delete_schedule/tool.go @@ -0,0 +1,44 @@ +package catalog_etl_delete_schedule + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the schedule was deleted"` + Error string `json:"error,omitempty" jsonschema:"error message if deletion failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_delete_schedule", + Description: "Deletes the synchronization schedule from an ETL integration.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + if err := clients.DeleteGenericSchedule(ctx, collibraClient, input.IngestibleId, input.Workflow); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to delete schedule: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_delete_schedule/tool_test.go b/pkg/tools/catalog_etl_delete_schedule/tool_test.go new file mode 100644 index 0000000..feb1ab8 --- /dev/null +++ b/pkg/tools/catalog_etl_delete_schedule/tool_test.go @@ -0,0 +1,60 @@ +package catalog_etl_delete_schedule_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/catalog_etl_delete_schedule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericDeleteSchedule(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusNoContent, nil + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestCatalogGenericDeleteScheduleAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericDeleteScheduleInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_get_all_schedules/tool.go b/pkg/tools/catalog_etl_get_all_schedules/tool.go new file mode 100644 index 0000000..902073a --- /dev/null +++ b/pkg/tools/catalog_etl_get_all_schedules/tool.go @@ -0,0 +1,45 @@ +package catalog_etl_get_all_schedules + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` +} + +type Output struct { + Schedule *clients.GenericSchedule `json:"schedule,omitempty" jsonschema:"the schedules for this integration"` + Found bool `json:"found" jsonschema:"whether schedules were found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_get_all_schedules", + Description: "Retrieves all synchronization schedules for an ETL integration.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schedule, err := clients.GetAllGenericSchedules(ctx, collibraClient, input.IngestibleId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get schedules: %s", err.Error())}, nil + } + return Output{Schedule: schedule, Found: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_get_all_schedules/tool_test.go b/pkg/tools/catalog_etl_get_all_schedules/tool_test.go new file mode 100644 index 0000000..4efb62c --- /dev/null +++ b/pkg/tools/catalog_etl_get_all_schedules/tool_test.go @@ -0,0 +1,64 @@ +package catalog_etl_get_all_schedules_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_get_all_schedules" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericGetAllSchedules(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedules", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{Id: 1, CronExpression: "0 2 * * *"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Schedule == nil || output.Schedule.Id != 1 { + t.Fatal("expected schedule id 1") + } +} + +func TestCatalogGenericGetAllSchedulesAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedules", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestCatalogGenericGetAllSchedulesInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_get_config/tool.go b/pkg/tools/catalog_etl_get_config/tool.go new file mode 100644 index 0000000..99c301b --- /dev/null +++ b/pkg/tools/catalog_etl_get_config/tool.go @@ -0,0 +1,45 @@ +package catalog_etl_get_config + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` +} + +type Output struct { + Config *clients.GenericConfiguration `json:"config,omitempty" jsonschema:"the integration configuration"` + Found bool `json:"found" jsonschema:"whether a configuration was found for this integration"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_get_config", + Description: "Retrieves the configuration for an ETL integration (e.g. Databricks Unity Catalog, Dataplex). The configuration value is a JSON string specific to the integration type.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + config, err := clients.GetGenericConfig(ctx, collibraClient, input.IngestibleId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get configuration: %s", err.Error())}, nil + } + return Output{Config: config, Found: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_get_config/tool_test.go b/pkg/tools/catalog_etl_get_config/tool_test.go new file mode 100644 index 0000000..2c56466 --- /dev/null +++ b/pkg/tools/catalog_etl_get_config/tool_test.go @@ -0,0 +1,68 @@ +package catalog_etl_get_config_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_get_config" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericGetConfig(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericConfiguration) { + return http.StatusOK, clients.GenericConfiguration{ + Id: "cfg-1", + IngestibleId: "00000000-0000-0000-0000-000000000001", + Value: `{"key":"value"}`, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Config == nil || output.Config.Id != "cfg-1" { + t.Fatal("expected config id cfg-1") + } +} + +func TestCatalogGenericGetConfigNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000002/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestCatalogGenericGetConfigInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_get_schedule/tool.go b/pkg/tools/catalog_etl_get_schedule/tool.go new file mode 100644 index 0000000..26bcdd7 --- /dev/null +++ b/pkg/tools/catalog_etl_get_schedule/tool.go @@ -0,0 +1,45 @@ +package catalog_etl_get_schedule + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` +} + +type Output struct { + Schedule *clients.GenericSchedule `json:"schedule,omitempty" jsonschema:"the integration schedule"` + Found bool `json:"found" jsonschema:"whether a schedule was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_get_schedule", + Description: "Retrieves the synchronization schedule for an ETL integration. Returns the cron expression, time zone, and timing of the last/next run.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schedule, err := clients.GetGenericSchedule(ctx, collibraClient, input.IngestibleId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get schedule: %s", err.Error())}, nil + } + return Output{Schedule: schedule, Found: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_get_schedule/tool_test.go b/pkg/tools/catalog_etl_get_schedule/tool_test.go new file mode 100644 index 0000000..9f7d23c --- /dev/null +++ b/pkg/tools/catalog_etl_get_schedule/tool_test.go @@ -0,0 +1,68 @@ +package catalog_etl_get_schedule_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_get_schedule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericGetSchedule(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{ + Id: 1, + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Schedule == nil || output.Schedule.CronExpression != "0 2 * * *" { + t.Fatal("expected cron expression 0 2 * * *") + } +} + +func TestCatalogGenericGetScheduleNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000002/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestCatalogGenericGetScheduleInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_get_schema/tool.go b/pkg/tools/catalog_etl_get_schema/tool.go new file mode 100644 index 0000000..05ebad1 --- /dev/null +++ b/pkg/tools/catalog_etl_get_schema/tool.go @@ -0,0 +1,45 @@ +package catalog_etl_get_schema + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` +} + +type Output struct { + Schema string `json:"schema,omitempty" jsonschema:"the data schema as a JSON string; describes the valid configuration fields for this integration"` + Found bool `json:"found" jsonschema:"whether a schema was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_get_schema", + Description: "Retrieves the data schema for an ETL integration. The schema describes the valid configuration fields that can be used with catalog_etl_save_generic_config.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schema, err := clients.GetGenericSchema(ctx, collibraClient, input.IngestibleId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get schema: %s", err.Error())}, nil + } + return Output{Schema: schema, Found: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_get_schema/tool_test.go b/pkg/tools/catalog_etl_get_schema/tool_test.go new file mode 100644 index 0000000..451f318 --- /dev/null +++ b/pkg/tools/catalog_etl_get_schema/tool_test.go @@ -0,0 +1,63 @@ +package catalog_etl_get_schema_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/catalog_etl_get_schema" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericGetSchema(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration/schema", testutil.StringHandlerOut(func(r *http.Request) (int, string) { + return http.StatusOK, `{"type":"object","properties":{"catalog":{"type":"string"}}}` + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Schema == "" { + t.Fatal("expected non-empty schema") + } +} + +func TestCatalogGenericGetSchemaNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000002/configuration/schema", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestCatalogGenericGetSchemaInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_save_generic_config/tool.go b/pkg/tools/catalog_etl_save_generic_config/tool.go new file mode 100644 index 0000000..9e71108 --- /dev/null +++ b/pkg/tools/catalog_etl_save_generic_config/tool.go @@ -0,0 +1,48 @@ +package catalog_etl_save_generic_config + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` + Configuration string `json:"configuration" jsonschema:"the configuration JSON string to save"` +} + +type Output struct { + Config *clients.GenericConfiguration `json:"config,omitempty" jsonschema:"the saved configuration"` + Success bool `json:"success" jsonschema:"whether the configuration was saved successfully"` + Error string `json:"error,omitempty" jsonschema:"error message if saving failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_save_generic_config", + Description: "Creates or updates the configuration for an ETL integration. The configuration is a JSON string whose schema depends on the integration type. Use catalog_etl_get_schema to discover valid fields.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + config, err := clients.SaveGenericConfig(ctx, collibraClient, input.IngestibleId, clients.SaveGenericConfigRequest{ + Configuration: input.Configuration, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to save configuration: %s", err.Error())}, nil + } + return Output{Config: config, Success: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_save_generic_config/tool_test.go b/pkg/tools/catalog_etl_save_generic_config/tool_test.go new file mode 100644 index 0000000..d9d159a --- /dev/null +++ b/pkg/tools/catalog_etl_save_generic_config/tool_test.go @@ -0,0 +1,66 @@ +package catalog_etl_save_generic_config_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_save_generic_config" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericSaveConfig(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerInOut(func(r *http.Request, in clients.SaveGenericConfigRequest) (int, clients.GenericConfiguration) { + return http.StatusOK, clients.GenericConfiguration{Id: "cfg-1", IngestibleId: "00000000-0000-0000-0000-000000000001", Value: in.Configuration} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + Configuration: `{"catalog":"my_catalog"}`, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } + if output.Config == nil || output.Config.Id != "cfg-1" { + t.Fatal("expected config id cfg-1") + } +} + +func TestCatalogGenericSaveConfigAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusBadRequest, "bad request" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + Configuration: "{}", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericSaveConfigInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad", Configuration: "{}"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_start_job/tool.go b/pkg/tools/catalog_etl_start_job/tool.go new file mode 100644 index 0000000..3a64072 --- /dev/null +++ b/pkg/tools/catalog_etl_start_job/tool.go @@ -0,0 +1,48 @@ +package catalog_etl_start_job + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` + CloudIngestionJobId string `json:"cloudIngestionJobId,omitempty" jsonschema:"optional cloud ingestion job ID to resume"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` + RuntimeArguments string `json:"runtimeArguments,omitempty" jsonschema:"optional runtime arguments as a string"` +} + +type Output struct { + Job *clients.GenericJob `json:"job,omitempty" jsonschema:"the started job details"` + Success bool `json:"success" jsonschema:"whether the job was started"` + Error string `json:"error,omitempty" jsonschema:"error message if starting failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_start_job", + Description: "Starts a synchronization job for an ETL integration.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + job, err := clients.StartGenericJob(ctx, collibraClient, input.IngestibleId, input.CloudIngestionJobId, input.Workflow, input.RuntimeArguments) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to start job: %s", err.Error())}, nil + } + return Output{Job: job, Success: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_start_job/tool_test.go b/pkg/tools/catalog_etl_start_job/tool_test.go new file mode 100644 index 0000000..2d0e40f --- /dev/null +++ b/pkg/tools/catalog_etl_start_job/tool_test.go @@ -0,0 +1,64 @@ +package catalog_etl_start_job_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_start_job" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericStartJob(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/run", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericJob) { + return http.StatusAccepted, clients.GenericJob{Id: "job-1", State: "RUNNING"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } + if output.Job == nil || output.Job.Id != "job-1" { + t.Fatal("expected job id job-1") + } +} + +func TestCatalogGenericStartJobAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/run", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusConflict, "job already running" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericStartJobInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_etl_update_schedule/tool.go b/pkg/tools/catalog_etl_update_schedule/tool.go new file mode 100644 index 0000000..51e3e8c --- /dev/null +++ b/pkg/tools/catalog_etl_update_schedule/tool.go @@ -0,0 +1,53 @@ +package catalog_etl_update_schedule + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` + CronExpression string `json:"cronExpression" jsonschema:"the updated cron expression (e.g. '0 3 * * *')"` + CronTimeZone string `json:"cronTimeZone" jsonschema:"the updated time zone for the cron schedule"` + CronJson string `json:"cronJson,omitempty" jsonschema:"optional JSON representation of the cron schedule"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` +} + +type Output struct { + Schedule *clients.GenericSchedule `json:"schedule,omitempty" jsonschema:"the updated schedule"` + Success bool `json:"success" jsonschema:"whether the schedule was updated successfully"` + Error string `json:"error,omitempty" jsonschema:"error message if the update failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_etl_update_schedule", + Description: "Updates the synchronization schedule for an ETL integration. Creates the schedule if it does not exist.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schedule, err := clients.UpdateGenericSchedule(ctx, collibraClient, input.IngestibleId, input.Workflow, clients.ChangeGenericScheduleRequest{ + CronExpression: input.CronExpression, + CronTimeZone: input.CronTimeZone, + CronJson: input.CronJson, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to update schedule: %s", err.Error())}, nil + } + return Output{Schedule: schedule, Success: true}, nil + } +} diff --git a/pkg/tools/catalog_etl_update_schedule/tool_test.go b/pkg/tools/catalog_etl_update_schedule/tool_test.go new file mode 100644 index 0000000..6e6494e --- /dev/null +++ b/pkg/tools/catalog_etl_update_schedule/tool_test.go @@ -0,0 +1,69 @@ +package catalog_etl_update_schedule_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_update_schedule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericUpdateSchedule(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ChangeGenericScheduleRequest) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{Id: 1, CronExpression: in.CronExpression} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + CronExpression: "0 3 * * *", + CronTimeZone: "UTC", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestCatalogGenericUpdateScheduleAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusBadRequest, "bad" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + CronExpression: "0 3 * * *", + CronTimeZone: "UTC", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericUpdateScheduleInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "bad", + CronExpression: "0 3 * * *", + CronTimeZone: "UTC", + }) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/configure_database_schemas/tool.go b/pkg/tools/configure_database_schemas/tool.go new file mode 100644 index 0000000..124c57a --- /dev/null +++ b/pkg/tools/configure_database_schemas/tool.go @@ -0,0 +1,186 @@ +// Package configure_database_schemas implements the configure_database_schemas MCP +// tool: discovers the schemas under an already-registered database and configures +// which schemas/tables get synchronized, via the public catalogDatabase API. +// +// This is the second of two steps in setting up a database for jdbc-ingestion — the +// first, registering the Database asset, is a separate tool (register_database), +// which returns the databaseConnectionId this tool takes as input. See +// register_database's package doc comment for why they're split rather than one call. +// +// Ambiguity guards: schemaNames is required when more than one schema is discovered, +// and include has no default — both force an explicit choice instead of silently +// syncing every schema/table. schemaNames/include still accept an explicit "*" to opt +// into "everything", but that must be a deliberate choice (confirmed with the user), +// never this tool's default behavior. Neither guard risks a dead end: schema discovery +// and selection have no side effects, so an ambiguity error here is a true no-op — the +// database registered by register_database is untouched, and retrying this tool call +// with the missing name(s) filled in just starts clean. +package configure_database_schemas + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// Sleep is used for polling backoff. Overridable in tests to avoid slow test runs. +var Sleep = time.Sleep + +const ( + // pollAttempts * pollInterval bounds the wait for the async refresh. A + // first-time connection to a data source can take 30-60s to establish, so + // this budget needs enough headroom to cover that cold-start case. + pollAttempts = 14 + pollInterval = 5 * time.Second +) + +type Input struct { + DatabaseConnectionID string `json:"databaseConnectionId" jsonschema:"The databaseConnectionId from register_database's Database output — identifies which registered database's schemas to configure."` + SchemaNames []string `json:"schemaNames,omitempty" jsonschema:"Exact names of the schemas to configure for synchronization, as they appear at the data source. Required if the database exposes more than one schema; if there is exactly one, it is selected automatically. Pass the single value '*' to configure every discovered schema — only do this after confirming with the user which schemas they actually want synced, never as a silent default when multiple schemas exist."` + + Include string `json:"include" jsonschema:"Comma-separated table name pattern to synchronize, '*' wildcard supported. Required — there is no default. Confirm with the user which tables they want before calling; do not pass '*' (all tables) without asking first. Applies to every schema selected via schemaNames."` + Exclude string `json:"exclude,omitempty" jsonschema:"Optional. Comma-separated table name pattern to exclude from synchronization."` + TargetDomainID string `json:"targetDomainId,omitempty" jsonschema:"Optional. UUID of a domain to create synchronized assets in. If omitted, an automatically created domain per schema is used."` + SkipViews bool `json:"skipViews,omitempty" jsonschema:"Optional. If true, database views are excluded from synchronization."` + RegisterSourceTags bool `json:"registerSourceTags,omitempty" jsonschema:"Optional. If true, registers tags from the data source (when supported by the driver)."` + IngestSemanticViews bool `json:"ingestSemanticViews,omitempty" jsonschema:"Optional. If true, ingests semantic views (when supported by the driver)."` + RegisterDataUsageStatistics bool `json:"registerDataUsageStatistics,omitempty" jsonschema:"Optional. If true, calculates data usage statistics for the synchronized tables (when supported by the data source)."` +} + +type Output struct { + SchemaConnections []clients.SchemaConnection `json:"schemaConnections,omitempty" jsonschema:"The schema connections discovered under the database, now configured for synchronization."` + Success bool `json:"success" jsonschema:"Whether the schemas were fully configured."` + Error string `json:"error,omitempty" jsonschema:"Error message if configuration failed or is still in progress (safe to retry)."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "configure_database_schemas", + Title: "Configure Database Schema Synchronization", + Description: "Discovers the schemas under a database registered via register_database and configures which schemas/tables get synchronized. Prerequisite for start_ingestion. If more than one schema is discovered, or include isn't provided, this returns an error naming the candidates instead of guessing — confirm the schemas and table pattern with the user, do not default to configuring everything. Safe to call again later to change which schemas/tables are synced.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("databaseConnectionId", input.DatabaseConnectionID); err != nil { + return Output{}, err + } + if err := validation.UUIDOptional("targetDomainId", input.TargetDomainID); err != nil { + return Output{}, err + } + if strings.TrimSpace(input.Include) == "" { + return Output{}, fmt.Errorf("include is required — confirm with the user which tables to synchronize (or that they want '*', all tables) before calling") + } + + discoveredSchemaConnections, err := discoverSchemaConnections(ctx, collibraClient, input.DatabaseConnectionID) + if err != nil { + return Output{Success: false, Error: err.Error()}, nil + } + + schemaConnections, err := selectSchemaConnections(discoveredSchemaConnections, input.SchemaNames) + if err != nil { + return Output{Success: false, Error: err.Error()}, nil + } + + configurations := make([]clients.SchemaMetadataConfiguration, len(schemaConnections)) + for i, schemaConnection := range schemaConnections { + configurations[i] = clients.SchemaMetadataConfiguration{ + SchemaConnectionID: schemaConnection.ID, + SynchronizationRules: []clients.MetadataSynchronizationRule{ + { + Include: input.Include, + Exclude: input.Exclude, + TargetDomainID: input.TargetDomainID, + SkipViews: input.SkipViews, + RegisterSourceTags: input.RegisterSourceTags, + IngestSemanticViews: input.IngestSemanticViews, + RegisterDataUsageStatistics: input.RegisterDataUsageStatistics, + }, + }, + } + } + + if _, err := clients.SetSchemaMetadataConfigurationsBatch(ctx, collibraClient, configurations); err != nil { + return Output{SchemaConnections: schemaConnections, Success: false, + Error: fmt.Sprintf("failed to set schema synchronization rules: %s", err.Error())}, nil + } + + return Output{SchemaConnections: schemaConnections, Success: true}, nil + } +} + +// discoverSchemaConnections triggers a refresh of schema connections for the given +// database connection and polls until at least one appears. +func discoverSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) ([]clients.SchemaConnection, error) { + if err := clients.RefreshSchemaConnections(ctx, client, databaseConnectionID); err != nil { + return nil, fmt.Errorf("failed to refresh schema connections: %w", err) + } + + for attempt := 0; attempt < pollAttempts; attempt++ { + if attempt > 0 { + Sleep(pollInterval) + } + found, err := clients.ListSchemaConnections(ctx, client, databaseConnectionID) + if err != nil { + return nil, fmt.Errorf("failed to list schema connections: %w", err) + } + if len(found) > 0 { + return found, nil + } + } + + return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or no capability referencing this connection exists yet — verify with edge_create_capability", databaseConnectionID, pollAttempts) +} + +// selectSchemaConnections narrows discovered to the schemas named in schemaNames. An +// empty schemaNames is only accepted when there's no ambiguity (a single discovered +// schema); with more than one discovered schema, the caller must either name the ones +// they want or pass ["*"] to explicitly opt into configuring all of them. +func selectSchemaConnections(discovered []clients.SchemaConnection, schemaNames []string) ([]clients.SchemaConnection, error) { + if len(schemaNames) == 1 && schemaNames[0] == "*" { + return discovered, nil + } + + if len(schemaNames) == 0 { + if len(discovered) > 1 { + names := make([]string, len(discovered)) + for i, s := range discovered { + names[i] = s.Name + } + return nil, fmt.Errorf("multiple schemas discovered (%s); specify schemaNames to select which to configure, or pass [\"*\"] to configure all of them", strings.Join(names, ", ")) + } + return discovered, nil + } + + byName := make(map[string]clients.SchemaConnection, len(discovered)) + for _, s := range discovered { + byName[s.Name] = s + } + + selected := make([]clients.SchemaConnection, 0, len(schemaNames)) + var missing []string + for _, name := range schemaNames { + s, ok := byName[name] + if !ok { + missing = append(missing, name) + continue + } + selected = append(selected, s) + } + if len(missing) > 0 { + return nil, fmt.Errorf("schema(s) not found among discovered schemas: %s", strings.Join(missing, ", ")) + } + + return selected, nil +} diff --git a/pkg/tools/configure_database_schemas/tool_test.go b/pkg/tools/configure_database_schemas/tool_test.go new file mode 100644 index 0000000..b845c49 --- /dev/null +++ b/pkg/tools/configure_database_schemas/tool_test.go @@ -0,0 +1,206 @@ +package configure_database_schemas_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/configure_database_schemas" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func init() { + tools.Sleep = func(time.Duration) {} +} + +func TestConfigureDatabase_Success(t *testing.T) { + dbConnID, _ := uuid.NewUUID() + schemaConnID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("databaseConnectionId") != dbConnID.String() { + t.Fatalf("unexpected databaseConnectionId: %s", r.URL.Query().Get("databaseConnectionId")) + } + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/schemaConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.SchemaConnection{ + {ID: schemaConnID.String(), Name: "public", DatabaseConnectionID: dbConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/schemaMetadataConfigurations/batch", testutil.JsonHandlerInOut(func(r *http.Request, in []clients.SchemaMetadataConfiguration) (int, []clients.SchemaMetadataConfiguration) { + if len(in) != 1 || in[0].SchemaConnectionID != schemaConnID.String() { + t.Fatalf("unexpected batch payload: %+v", in) + } + if len(in[0].SynchronizationRules) != 1 || in[0].SynchronizationRules[0].Include != "*" { + t.Fatalf("unexpected synchronization rules: %+v", in[0].SynchronizationRules) + } + return http.StatusCreated, in + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseConnectionID: dbConnID.String(), + Include: "*", + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if len(output.SchemaConnections) != 1 { + t.Fatalf("expected 1 schema connection, got %d", len(output.SchemaConnections)) + } +} + +func TestConfigureDatabase_MissingInclude(t *testing.T) { + dbConnID, _ := uuid.NewUUID() + + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseConnectionID: dbConnID.String(), + }) + if err == nil { + t.Fatalf("expected an error when include is omitted") + } +} + +func TestConfigureDatabase_MultipleSchemasRequireSchemaNames(t *testing.T) { + dbConnID, _ := uuid.NewUUID() + publicSchemaID, _ := uuid.NewUUID() + privateSchemaID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/schemaConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.SchemaConnection{ + {ID: publicSchemaID.String(), Name: "public", DatabaseConnectionID: dbConnID.String()}, + {ID: privateSchemaID.String(), Name: "private", DatabaseConnectionID: dbConnID.String()}, + }, + } + })) + batchCalled := false + mux.Handle("POST /rest/catalogDatabase/v1/schemaMetadataConfigurations/batch", testutil.JsonHandlerInOut(func(r *http.Request, in []clients.SchemaMetadataConfiguration) (int, []clients.SchemaMetadataConfiguration) { + batchCalled = true + return http.StatusCreated, in + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseConnectionID: dbConnID.String(), + Include: "*", + }) + if err != nil { + t.Fatalf("expected no error (failures reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when multiple schemas are discovered without schemaNames") + } + if output.Error == "" { + t.Fatalf("expected an error message naming the discovered schemas") + } + if batchCalled { + t.Fatalf("expected no synchronization rules to be set when schema selection fails") + } +} + +func TestConfigureDatabase_SchemaNamesSelectsSubset(t *testing.T) { + dbConnID, _ := uuid.NewUUID() + publicSchemaID, _ := uuid.NewUUID() + privateSchemaID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/schemaConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.SchemaConnection{ + {ID: publicSchemaID.String(), Name: "public", DatabaseConnectionID: dbConnID.String()}, + {ID: privateSchemaID.String(), Name: "private", DatabaseConnectionID: dbConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/schemaMetadataConfigurations/batch", testutil.JsonHandlerInOut(func(r *http.Request, in []clients.SchemaMetadataConfiguration) (int, []clients.SchemaMetadataConfiguration) { + if len(in) != 1 || in[0].SchemaConnectionID != publicSchemaID.String() { + t.Fatalf("unexpected batch payload: %+v", in) + } + return http.StatusCreated, in + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseConnectionID: dbConnID.String(), + SchemaNames: []string{"public"}, + Include: "*", + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if len(output.SchemaConnections) != 1 || output.SchemaConnections[0].Name != "public" { + t.Fatalf("expected only the 'public' schema to be configured, got: %+v", output.SchemaConnections) + } +} + +func TestConfigureDatabase_NoSchemaConnectionsDiscovered(t *testing.T) { + dbConnID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/schemaConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{"results": []clients.SchemaConnection{}} + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseConnectionID: dbConnID.String(), + Include: "*", + }) + if err != nil { + t.Fatalf("expected no error (failures reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when no schema connections are discovered") + } + if output.Error == "" { + t.Fatalf("expected an error message") + } +} + +func TestConfigureDatabase_InvalidInput(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseConnectionID: "not-a-uuid", + Include: "*", + }) + if err == nil { + t.Fatalf("expected an error for invalid databaseConnectionId") + } +} diff --git a/pkg/tools/create_community/tool.go b/pkg/tools/create_community/tool.go new file mode 100644 index 0000000..86f028d --- /dev/null +++ b/pkg/tools/create_community/tool.go @@ -0,0 +1,49 @@ +// Package create_community implements the create_community MCP tool: creates a DGC +// community, the top-level catalog scaffolding required before creating domains and +// registering databases via register_database. +package create_community + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name" jsonschema:"The community name."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the community."` +} + +type Output struct { + Community *clients.CommunityDetails `json:"community,omitempty" jsonschema:"The created community."` + Success bool `json:"success" jsonschema:"Whether the community was created successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if creating the community failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "create_community", + Title: "Create Community", + Description: "Creates a DGC community. A community is required before creating a domain (create_domain) and registering a database (register_database).", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + community, err := clients.CreateCommunity(ctx, collibraClient, clients.CreateCommunityRequest{ + Name: input.Name, + Description: input.Description, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to create community: %s", err.Error())}, nil + } + return Output{Community: community, Success: true}, nil + } +} diff --git a/pkg/tools/create_community/tool_test.go b/pkg/tools/create_community/tool_test.go new file mode 100644 index 0000000..2fed332 --- /dev/null +++ b/pkg/tools/create_community/tool_test.go @@ -0,0 +1,61 @@ +package create_community_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/create_community" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestCreateCommunity(t *testing.T) { + communityID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /rest/2.0/communities", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CreateCommunityRequest) (int, clients.CommunityDetails) { + if in.Name != "Local Development" { + t.Fatalf("unexpected name: %s", in.Name) + } + return http.StatusCreated, clients.CommunityDetails{ID: communityID.String(), Name: in.Name, Description: in.Description} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "Local Development", + Description: "Local e2e test environment", + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Community.ID != communityID.String() { + t.Fatalf("expected id %s, got %s", communityID.String(), output.Community.ID) + } +} + +func TestCreateCommunity_Failure(t *testing.T) { + handler := http.NewServeMux() + handler.HandleFunc("POST /rest/2.0/communities", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Name: "bad"}) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure") + } +} diff --git a/pkg/tools/create_domain/tool.go b/pkg/tools/create_domain/tool.go new file mode 100644 index 0000000..f5f4a4b --- /dev/null +++ b/pkg/tools/create_domain/tool.go @@ -0,0 +1,63 @@ +// Package create_domain implements the create_domain MCP tool: creates a DGC domain +// within a community, e.g. a "Physical Data Dictionary" domain required before +// registering a database via register_database. +package create_domain + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name" jsonschema:"The domain name."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the domain."` + CommunityID string `json:"communityId" jsonschema:"UUID of the community (created via create_community) this domain belongs to."` + TypeID string `json:"typeId" jsonschema:"UUID of the domain type. Use find_domain_types to resolve a name (e.g. 'Physical Data Dictionary', used for jdbc-ingestion databases) to its UUID."` + ExcludedFromAutoHyperlinking bool `json:"excludedFromAutoHyperlinking,omitempty" jsonschema:"Optional. If true, assets in this domain are excluded from automatic hyperlinking."` +} + +type Output struct { + Domain *clients.DomainDetails `json:"domain,omitempty" jsonschema:"The created domain."` + Success bool `json:"success" jsonschema:"Whether the domain was created successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if creating the domain failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "create_domain", + Title: "Create Domain", + Description: "Creates a DGC domain within a community, e.g. a Physical Data Dictionary domain required before registering a database via register_database.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("communityId", input.CommunityID); err != nil { + return Output{}, err + } + if err := validation.UUID("typeId", input.TypeID); err != nil { + return Output{}, err + } + + domain, err := clients.CreateDomain(ctx, collibraClient, clients.CreateDomainRequest{ + Name: input.Name, + Description: input.Description, + CommunityID: input.CommunityID, + TypeID: input.TypeID, + ExcludedFromAutoHyperlinking: input.ExcludedFromAutoHyperlinking, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to create domain: %s", err.Error())}, nil + } + return Output{Domain: domain, Success: true}, nil + } +} diff --git a/pkg/tools/create_domain/tool_test.go b/pkg/tools/create_domain/tool_test.go new file mode 100644 index 0000000..b54590d --- /dev/null +++ b/pkg/tools/create_domain/tool_test.go @@ -0,0 +1,62 @@ +package create_domain_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/create_domain" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestCreateDomain(t *testing.T) { + communityID, _ := uuid.NewUUID() + typeID, _ := uuid.NewUUID() + domainID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /rest/2.0/domains", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CreateDomainRequest) (int, clients.DomainDetails) { + if in.CommunityID != communityID.String() { + t.Fatalf("unexpected communityId: %s", in.CommunityID) + } + return http.StatusCreated, clients.DomainDetails{ + ID: domainID.String(), + Name: in.Name, + CommunityID: in.CommunityID, + TypeID: in.TypeID, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "postgres-src [Physical Data Dictionary]", + CommunityID: communityID.String(), + TypeID: typeID.String(), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Domain.ID != domainID.String() { + t.Fatalf("expected id %s, got %s", domainID.String(), output.Domain.ID) + } +} + +func TestCreateDomain_InvalidCommunityID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "bad", + CommunityID: "not-a-uuid", + TypeID: "also-not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid communityId") + } +} diff --git a/pkg/tools/edge_cancel_job/tool.go b/pkg/tools/edge_cancel_job/tool.go new file mode 100644 index 0000000..1e78e77 --- /dev/null +++ b/pkg/tools/edge_cancel_job/tool.go @@ -0,0 +1,43 @@ +package edge_cancel_job + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the Edge job to cancel"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the cancellation request was accepted"` + Error string `json:"error,omitempty" jsonschema:"error message if cancellation failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_cancel_job", + Description: "Stops a running Edge job by its UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + if err := clients.CancelEdgeJob(ctx, collibraClient, input.Id); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to cancel job: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/edge_cancel_job/tool_test.go b/pkg/tools/edge_cancel_job/tool_test.go new file mode 100644 index 0000000..b4e9cef --- /dev/null +++ b/pkg/tools/edge_cancel_job/tool_test.go @@ -0,0 +1,60 @@ +package edge_cancel_job_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/edge_cancel_job" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeCancelJob(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/jobs/00000000-0000-0000-0000-000000000001/cancel", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestEdgeCancelJobAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/jobs/00000000-0000-0000-0000-000000000001/cancel", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "job not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestEdgeCancelJobInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_create_capability/tool.go b/pkg/tools/edge_create_capability/tool.go new file mode 100644 index 0000000..d3610e2 --- /dev/null +++ b/pkg/tools/edge_create_capability/tool.go @@ -0,0 +1,78 @@ +// Package edge_create_capability implements the edge_create_capability MCP tool: creates or +// updates an Edge capability (e.g. jdbc-ingestion or a lineage capability) via the +// private Edge capability management API. +package edge_create_capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + CapabilityID string `json:"capabilityId,omitempty" jsonschema:"Optional. UUID of the capability to create or update. If provided, the capability is created with this exact id (or updated if it already exists). If omitted, a new capability is created and the server assigns an id."` + Name string `json:"name" jsonschema:"The capability name."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the capability."` + TypeID string `json:"typeId" jsonschema:"The id of the capability type (e.g. 'jdbc-ingestion'). Use the edge_list_capability_types tool to discover available types and their expected parameters."` + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this capability will run. Use the edge_list_sites tool to discover available sites."` + Parameters map[string]any `json:"parameters" jsonschema:"Capability install parameters as defined by the capability type's manifest. For jdbc-ingestion, this includes 'connection' (the id of a connection created via edge_create_connection), 'data-source-type', 'message-mode', and an optional 'other-settings' list of {name, type, value}."` +} + +type Output struct { + Capability *clients.EdgeCapability `json:"capability,omitempty" jsonschema:"The created or updated capability."` + Success bool `json:"success" jsonschema:"Whether the capability was saved successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if saving the capability failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_create_capability", + Title: "Create or Update Edge Capability", + Description: "Creates or updates an Edge capability (e.g. jdbc-ingestion) via the private Edge capability management API. Does not run the capability — use start_ingestion to trigger a jdbc-ingestion run.", + Handler: handler(collibraClient), + Permissions: []string{"dgc.edge-integration-capability-manage"}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUIDOptional("capabilityId", input.CapabilityID); err != nil { + return Output{}, err + } + if err := validation.UUID("edgeSiteId", input.EdgeSiteID); err != nil { + return Output{}, err + } + + parameters := input.Parameters + if parameters == nil { + parameters = map[string]any{} + } + + request := clients.CapabilityRequest{ + TypeID: input.TypeID, + Name: input.Name, + Description: input.Description, + EdgeSiteID: input.EdgeSiteID, + Parameters: parameters, + } + + var capability *clients.EdgeCapability + var err error + if input.CapabilityID != "" { + capability, err = clients.CreateOrUpdateCapability(ctx, collibraClient, input.CapabilityID, request) + } else { + capability, err = clients.CreateCapability(ctx, collibraClient, request) + } + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to save capability: %s", err.Error())}, nil + } + + return Output{Capability: capability, Success: true}, nil + } +} diff --git a/pkg/tools/edge_create_capability/tool_test.go b/pkg/tools/edge_create_capability/tool_test.go new file mode 100644 index 0000000..444c648 --- /dev/null +++ b/pkg/tools/edge_create_capability/tool_test.go @@ -0,0 +1,95 @@ +package edge_create_capability_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_create_capability" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestCreateCapability_Create(t *testing.T) { + siteID, _ := uuid.NewUUID() + capID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/capabilities", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityRequest) (int, clients.EdgeCapability) { + if in.TypeID != "jdbc-ingestion" { + t.Fatalf("unexpected typeId: %s", in.TypeID) + } + return http.StatusCreated, clients.EdgeCapability{ + Id: capID.String(), + Name: in.Name, + EdgeSiteId: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "local-jdbc-ingestion", + TypeID: "jdbc-ingestion", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{"connection": "some-connection-id", "message-mode": "KAFKA"}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Capability.Id != capID.String() { + t.Fatalf("expected id %s, got %s", capID.String(), output.Capability.Id) + } +} + +func TestCreateCapability_UpdateWithKnownID(t *testing.T) { + siteID, _ := uuid.NewUUID() + capID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("PUT /edge/api/rest/v2/capabilities/"+capID.String(), testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityRequest) (int, clients.EdgeCapability) { + return http.StatusOK, clients.EdgeCapability{ + Id: capID.String(), + Name: in.Name, + EdgeSiteId: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + CapabilityID: capID.String(), + Name: "local-jdbc-ingestion", + TypeID: "jdbc-ingestion", + EdgeSiteID: siteID.String(), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestCreateCapability_InvalidCapabilityID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + CapabilityID: "not-a-uuid", + Name: "bad", + TypeID: "jdbc-ingestion", + EdgeSiteID: "also-not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid capabilityId") + } +} diff --git a/pkg/tools/edge_create_connection/tool.go b/pkg/tools/edge_create_connection/tool.go new file mode 100644 index 0000000..dbf2f48 --- /dev/null +++ b/pkg/tools/edge_create_connection/tool.go @@ -0,0 +1,122 @@ +// Package edge_create_connection implements the edge_create_connection MCP tool: creates or +// updates an Edge connection (e.g. a JDBC connection backing the jdbc-ingestion +// capability) via the private Edge connection management API. +package edge_create_connection + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + ConnectionID string `json:"connectionId,omitempty" jsonschema:"Optional. UUID of the connection to create or update. If provided, the connection is created with this exact id (or updated if it already exists). If omitted, a new connection is created and the server assigns an id."` + Name string `json:"name" jsonschema:"The connection name."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the connection."` + TypeID string `json:"typeId" jsonschema:"The id of the connection type (e.g. 'Generic' for a generic JDBC connection, or a vendor-specific type such as a Snowflake connection type). Determines which parameters are expected — use edge_list_capability_types to inspect a type's manifest."` + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this connection will be valid. Use the edge_list_sites tool to discover available sites."` + VaultID string `json:"vaultId,omitempty" jsonschema:"Optional UUID of the vault to retrieve vault-backed parameters from."` + Parameters map[string]any `json:"parameters" jsonschema:"Fixed connection parameters as defined by the connection type's manifest (e.g. driver-class, connection-string for the 'Generic' JDBC connection type). Do NOT put open-ended/vendor-specific properties (e.g. a Snowflake connection's Role, Warehouse, User, Database, or a private key file) here — use additionalProperties instead. If you don't already know a data source's driver class, connection string format, required properties, or verified driver source, call get_data_source_setup_guide first instead of guessing. For 'driver-jar' (and any other FILE-type parameter), the value is an artifact URI from upload_file — see that tool's policy on where driver files may come from."` + + AdditionalProperties []AdditionalProperty `json:"additionalProperties,omitempty" jsonschema:"Optional. Open-ended, connection-type-defined properties beyond the fixed manifest parameters — e.g. a Snowflake connection (via the 'Generic' JDBC connection type) uses this for Role, Warehouse, User, Database, and private_key_file. Confirmed shape: each entry is injected as {name, type, value, secret} into a single array-valued parameter (see additionalPropertiesKey). For a FILE-type property, first call upload_file and use its returned artifact URI as this entry's value with type='file'."` + AdditionalPropertiesKey string `json:"additionalPropertiesKey,omitempty" jsonschema:"Optional. The manifest parameter name that additionalProperties are grouped under. Defaults to 'connection-properties', which is correct for the 'Generic' JDBC connection type (confirmed live). Other connection types may use a different key (e.g. AWS/GCP connections use 'additional-parameters') — check the type's manifest via edge_list_capability_types if unsure."` +} + +// AdditionalProperty is one entry of an open-ended, connection-type-defined property +// list (DGC's ConnectionManifest.ParameterType.USER_DEFINED). Confirmed against a real +// working Snowflake-via-Generic-JDBC connection: each entry is {name, type, value, +// secret}, and a FILE-type entry's value is the bare artifact URI (same shape as any +// other FILE parameter), not a wrapped object. +type AdditionalProperty struct { + Name string `json:"name" jsonschema:"The property name (e.g. 'Role', 'Warehouse', 'private_key_file')."` + Type string `json:"type,omitempty" jsonschema:"'string' (default) or 'file'. Use 'file' when value is an artifact URI returned by upload_file."` + Value string `json:"value" jsonschema:"The property value. For type='file', this is the artifact URI returned by upload_file, not raw file content."` + Secret bool `json:"secret,omitempty" jsonschema:"Optional. Whether this value should be treated as a secret (e.g. a private key file or password)."` +} + +type Output struct { + Connection *clients.EdgeConnection `json:"connection,omitempty" jsonschema:"The created or updated connection."` + Success bool `json:"success" jsonschema:"Whether the connection was saved successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if saving the connection failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_create_connection", + Title: "Create or Update Edge Connection", + Description: "Creates or updates an Edge connection (e.g. a JDBC connection for the jdbc-ingestion capability) via the private Edge connection management API. Does not download driver files itself — see upload_file for how driver jars must be sourced (Collibra Marketplace only, not arbitrary URLs).", + Handler: handler(collibraClient), + Permissions: []string{"dgc.edge-integration-capability-manage"}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUIDOptional("connectionId", input.ConnectionID); err != nil { + return Output{}, err + } + if err := validation.UUID("edgeSiteId", input.EdgeSiteID); err != nil { + return Output{}, err + } + if err := validation.UUIDOptional("vaultId", input.VaultID); err != nil { + return Output{}, err + } + + parameters := input.Parameters + if parameters == nil { + parameters = map[string]any{} + } + + if len(input.AdditionalProperties) > 0 { + key := input.AdditionalPropertiesKey + if key == "" { + key = "connection-properties" + } + entries := make([]map[string]any, len(input.AdditionalProperties)) + for i, prop := range input.AdditionalProperties { + if prop.Name == "" { + return Output{}, fmt.Errorf("additionalProperties[%d]: name is required", i) + } + propType := prop.Type + if propType == "" { + propType = "string" + } + entries[i] = map[string]any{ + "name": prop.Name, + "type": propType, + "value": prop.Value, + "secret": prop.Secret, + } + } + parameters[key] = entries + } + + request := clients.ConnectionRequest{ + Name: input.Name, + Description: input.Description, + TypeID: input.TypeID, + EdgeSiteID: input.EdgeSiteID, + VaultID: input.VaultID, + Parameters: parameters, + } + + var connection *clients.EdgeConnection + var err error + if input.ConnectionID != "" { + connection, err = clients.CreateOrUpdateConnection(ctx, collibraClient, input.ConnectionID, request) + } else { + connection, err = clients.CreateConnection(ctx, collibraClient, request) + } + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to save connection: %s", err.Error())}, nil + } + + return Output{Connection: connection, Success: true}, nil + } +} diff --git a/pkg/tools/edge_create_connection/tool_test.go b/pkg/tools/edge_create_connection/tool_test.go new file mode 100644 index 0000000..cf1dd8a --- /dev/null +++ b/pkg/tools/edge_create_connection/tool_test.go @@ -0,0 +1,223 @@ +package edge_create_connection_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_create_connection" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestCreateConnection_Create(t *testing.T) { + siteID, _ := uuid.NewUUID() + connID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { + if in.Name != "local-postgres-source" { + t.Fatalf("unexpected name: %s", in.Name) + } + if in.EdgeSiteID != siteID.String() { + t.Fatalf("unexpected edgeSiteId: %s", in.EdgeSiteID) + } + return http.StatusCreated, clients.EdgeConnection{ + Id: connID.String(), + Name: in.Name, + TypeId: in.TypeID, + EdgeSiteId: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "local-postgres-source", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{"driver-class": "org.postgresql.Driver"}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Connection.Id != connID.String() { + t.Fatalf("expected id %s, got %s", connID.String(), output.Connection.Id) + } +} + +func TestCreateConnection_UpdateWithKnownID(t *testing.T) { + siteID, _ := uuid.NewUUID() + connID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("PUT /edge/api/rest/v2/connections/"+connID.String(), testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { + return http.StatusOK, clients.EdgeConnection{ + Id: connID.String(), + Name: in.Name, + TypeId: in.TypeID, + EdgeSiteId: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + ConnectionID: connID.String(), + Name: "local-postgres-source", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Connection.Id != connID.String() { + t.Fatalf("expected id %s, got %s", connID.String(), output.Connection.Id) + } +} + +func TestCreateConnection_InvalidEdgeSiteID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "bad", + TypeID: "Generic", + EdgeSiteID: "not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid edgeSiteId") + } +} + +func TestCreateConnection_AdditionalProperties(t *testing.T) { + // Mirrors a real, confirmed-working Snowflake-via-Generic-JDBC connection's + // parameters: driver-class/connection-string/driver-jar fixed, plus a + // "connection-properties" array of {name, type, value, secret} for Role, + // Warehouse, private_key_file, User, Database. + siteID, _ := uuid.NewUUID() + connID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { + props, ok := in.Parameters["connection-properties"].([]any) + if !ok || len(props) != 2 { + t.Fatalf("expected 2 connection-properties entries, got: %#v", in.Parameters["connection-properties"]) + } + role, ok := props[0].(map[string]any) + if !ok || role["name"] != "Role" || role["type"] != "string" || role["value"] != "SOFTWARE_DEVELOPMENT" || role["secret"] != false { + t.Fatalf("unexpected Role entry: %#v", props[0]) + } + keyFile, ok := props[1].(map[string]any) + if !ok || keyFile["name"] != "private_key_file" || keyFile["type"] != "file" || keyFile["value"] != "artifact://uuid/key.p8" || keyFile["secret"] != true { + t.Fatalf("unexpected private_key_file entry: %#v", props[1]) + } + return http.StatusCreated, clients.EdgeConnection{Id: connID.String(), Name: in.Name, EdgeSiteId: in.EdgeSiteID, Parameters: in.Parameters} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "snowflake-jdbc", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{ + "driver-class": "net.snowflake.client.jdbc.SnowflakeDriver", + "connection-string": "jdbc:snowflake://acme.snowflakecomputing.com", + }, + AdditionalProperties: []tools.AdditionalProperty{ + {Name: "Role", Value: "SOFTWARE_DEVELOPMENT"}, + {Name: "private_key_file", Type: "file", Value: "artifact://uuid/key.p8", Secret: true}, + }, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestCreateConnection_AdditionalProperties_CustomKey(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { + if _, ok := in.Parameters["additional-parameters"]; !ok { + t.Fatalf("expected additional-parameters key, got parameters: %#v", in.Parameters) + } + return http.StatusCreated, clients.EdgeConnection{Parameters: in.Parameters} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "aws-conn", + TypeID: "AWS", + EdgeSiteID: siteID.String(), + AdditionalPropertiesKey: "additional-parameters", + AdditionalProperties: []tools.AdditionalProperty{{Name: "region", Value: "us-east-1"}}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +// edge_create_connection deliberately does not fetch driver files from arbitrary URLs +// (Collibra Marketplace requires an authenticated download — see upload_file's +// policy doc) — the only supported path is referencing an artifact URI that was +// already uploaded via upload_file, passed through like any other parameter. +func TestCreateConnection_PreUploadedDriverJar(t *testing.T) { + siteID, _ := uuid.NewUUID() + connID, _ := uuid.NewUUID() + const uploadedJarURI = "jar://test-uuid/driver.jar" + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { + if in.Parameters["driver-jar"] != uploadedJarURI { + t.Fatalf("expected driver-jar %q, got %v", uploadedJarURI, in.Parameters["driver-jar"]) + } + return http.StatusCreated, clients.EdgeConnection{ + Id: connID.String(), + Name: in.Name, + EdgeSiteId: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "local-postgres-source", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{"driver-jar": uploadedJarURI}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} diff --git a/pkg/tools/edge_delete_capability/tool.go b/pkg/tools/edge_delete_capability/tool.go new file mode 100644 index 0000000..42595fb --- /dev/null +++ b/pkg/tools/edge_delete_capability/tool.go @@ -0,0 +1,43 @@ +package edge_delete_capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the capability to delete"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the capability was deleted"` + Error string `json:"error,omitempty" jsonschema:"error message if deletion failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_delete_capability", + Description: "Deletes an Edge capability by its UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + if err := clients.DeleteCapability(ctx, collibraClient, input.Id); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to delete capability: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/edge_delete_capability/tool_test.go b/pkg/tools/edge_delete_capability/tool_test.go new file mode 100644 index 0000000..b936c08 --- /dev/null +++ b/pkg/tools/edge_delete_capability/tool_test.go @@ -0,0 +1,60 @@ +package edge_delete_capability_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/edge_delete_capability" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeDeleteCapability(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestEdgeDeleteCapabilityAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000002", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestEdgeDeleteCapabilityInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_delete_connection/tool.go b/pkg/tools/edge_delete_connection/tool.go new file mode 100644 index 0000000..5e7e03e --- /dev/null +++ b/pkg/tools/edge_delete_connection/tool.go @@ -0,0 +1,43 @@ +package edge_delete_connection + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + ConnectionId string `json:"connectionId" jsonschema:"the UUID of the connection to delete"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the connection was deleted"` + Error string `json:"error,omitempty" jsonschema:"error message if deletion failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_delete_connection", + Description: "Deletes an Edge connection by its UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("connectionId", input.ConnectionId); err != nil { + return Output{}, err + } + if err := clients.DeleteConnection(ctx, collibraClient, input.ConnectionId); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to delete connection: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/edge_delete_connection/tool_test.go b/pkg/tools/edge_delete_connection/tool_test.go new file mode 100644 index 0000000..49e79a8 --- /dev/null +++ b/pkg/tools/edge_delete_connection/tool_test.go @@ -0,0 +1,60 @@ +package edge_delete_connection_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/edge_delete_connection" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeDeleteConnection(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + ConnectionId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestEdgeDeleteConnectionAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + ConnectionId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestEdgeDeleteConnectionInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ConnectionId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_find_capabilities/tool.go b/pkg/tools/edge_find_capabilities/tool.go new file mode 100644 index 0000000..93287f6 --- /dev/null +++ b/pkg/tools/edge_find_capabilities/tool.go @@ -0,0 +1,54 @@ +package edge_find_capabilities + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + EdgeSiteId string `json:"edgeSiteId,omitempty" jsonschema:"optional UUID of the edge site to filter capabilities by"` + Labels map[string]string `json:"labels,omitempty" jsonschema:"optional label key-value pairs to filter by; multiple labels are ANDed"` + Parameters map[string]string `json:"parameters,omitempty" jsonschema:"optional parameter key-value pairs to filter by (e.g. connection UUID)"` +} + +type Output struct { + Capabilities []clients.EdgeCapability `json:"capabilities" jsonschema:"list of matching capabilities"` + Count int `json:"count" jsonschema:"number of matching capabilities"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_find_capabilities", + Description: "Finds Edge capabilities matching the given criteria. All filter fields are ANDed; map/list field values are ORed. Useful for finding capabilities by edge site or parameter value.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUIDOptional("edgeSiteId", input.EdgeSiteId); err != nil { + return Output{}, err + } + caps, err := clients.FindCapabilities(ctx, collibraClient, clients.CapabilityFindRequest{ + EdgeSiteId: input.EdgeSiteId, + Labels: input.Labels, + Parameters: input.Parameters, + }) + if err != nil { + return Output{Capabilities: []clients.EdgeCapability{}, Error: fmt.Sprintf("failed to find capabilities: %s", err.Error())}, nil + } + if caps == nil { + caps = []clients.EdgeCapability{} + } + return Output{Capabilities: caps, Count: len(caps)}, nil + } +} diff --git a/pkg/tools/edge_find_capabilities/tool_test.go b/pkg/tools/edge_find_capabilities/tool_test.go new file mode 100644 index 0000000..991485f --- /dev/null +++ b/pkg/tools/edge_find_capabilities/tool_test.go @@ -0,0 +1,66 @@ +package edge_find_capabilities_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_find_capabilities" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeFindCapabilities(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/find", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityFindRequest) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "cap-1", Name: "Databricks Cap", EdgeSiteId: in.EdgeSiteId}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + EdgeSiteId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + if len(output.Capabilities) != 1 || output.Capabilities[0].Id != "cap-1" { + t.Fatalf("expected cap-1, got %+v", output.Capabilities) + } +} + +func TestEdgeFindCapabilitiesAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/find", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "error" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error") + } +} + +func TestEdgeFindCapabilitiesInvalidSiteId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + EdgeSiteId: "not-a-uuid", + }) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_find_connections/tool.go b/pkg/tools/edge_find_connections/tool.go new file mode 100644 index 0000000..7d780e0 --- /dev/null +++ b/pkg/tools/edge_find_connections/tool.go @@ -0,0 +1,56 @@ +// Package edge_find_connections implements the edge_find_connections MCP tool: finds Edge +// connections by name. Primarily for picking up a connection the user created +// manually (e.g. via the DGC/Edge UI, when a driver file was too large to upload +// through upload_file's contentBase64) — see edge_create_connection's package doc for the +// guided-manual-creation flow this supports. +package edge_find_connections + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + EdgeSiteID string `json:"edgeSiteId,omitempty" jsonschema:"Optional. UUID of the edge site to restrict the search to."` + Name string `json:"name,omitempty" jsonschema:"Optional. The connection name to search for."` + NameMatchMode string `json:"nameMatchMode,omitempty" jsonschema:"Optional. 'ANYWHERE' (default, substring match) or 'EXACT'."` +} + +type Output struct { + Connections []clients.EdgeConnection `json:"connections" jsonschema:"The matching connections."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_find_connections", + Title: "Find Edge Connections", + Description: "Finds Edge connections by name (and optionally edge site). Use this to pick up a connection that already exists — e.g. one the user created manually via the DGC/Edge UI for a driver file too large to pass through upload_file — instead of creating a duplicate with edge_create_connection.", + Handler: handler(collibraClient), + Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUIDOptional("edgeSiteId", input.EdgeSiteID); err != nil { + return Output{}, err + } + + connections, err := clients.FindConnections(ctx, collibraClient, clients.ConnectionFindRequest{ + EdgeSiteID: input.EdgeSiteID, + Name: input.Name, + NameMatchMode: input.NameMatchMode, + }) + if err != nil { + return Output{}, err + } + + return Output{Connections: connections}, nil + } +} diff --git a/pkg/tools/edge_find_connections/tool_test.go b/pkg/tools/edge_find_connections/tool_test.go new file mode 100644 index 0000000..171e50b --- /dev/null +++ b/pkg/tools/edge_find_connections/tool_test.go @@ -0,0 +1,47 @@ +package edge_find_connections_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_find_connections" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindConnections(t *testing.T) { + connID, _ := uuid.NewUUID() + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections/find", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionFindRequest) (int, []clients.EdgeConnection) { + if in.Name != "snowflake-manual" { + t.Fatalf("unexpected name: %s", in.Name) + } + return http.StatusOK, []clients.EdgeConnection{ + {Id: connID.String(), Name: "snowflake-manual", EdgeSiteId: siteID.String()}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Name: "snowflake-manual"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.Connections) != 1 || output.Connections[0].Id != connID.String() { + t.Fatalf("unexpected output: %+v", output) + } +} + +func TestFindConnections_InvalidEdgeSiteID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid edgeSiteId") + } +} diff --git a/pkg/tools/edge_get_capability/tool.go b/pkg/tools/edge_get_capability/tool.go new file mode 100644 index 0000000..76003ad --- /dev/null +++ b/pkg/tools/edge_get_capability/tool.go @@ -0,0 +1,45 @@ +package edge_get_capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the capability to retrieve"` +} + +type Output struct { + Capability *clients.EdgeCapability `json:"capability,omitempty" jsonschema:"the capability details"` + Found bool `json:"found" jsonschema:"whether the capability was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_get_capability", + Description: "Gets an Edge capability by its UUID. Returns the capability's configuration, type, edge site, and parameters.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + cap, err := clients.GetCapability(ctx, collibraClient, input.Id) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get capability: %s", err.Error())}, nil + } + return Output{Capability: cap, Found: true}, nil + } +} diff --git a/pkg/tools/edge_get_capability/tool_test.go b/pkg/tools/edge_get_capability/tool_test.go new file mode 100644 index 0000000..4a64c3f --- /dev/null +++ b/pkg/tools/edge_get_capability/tool_test.go @@ -0,0 +1,67 @@ +package edge_get_capability_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_get_capability" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeGetCapability(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.EdgeCapability) { + return http.StatusOK, clients.EdgeCapability{Id: "00000000-0000-0000-0000-000000000001", Name: "Test Cap"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Capability == nil || output.Capability.Name != "Test Cap" { + t.Fatal("expected capability name Test Cap") + } +} + +func TestEdgeGetCapabilityNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000002", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } + if output.Error == "" { + t.Fatal("expected error message") + } +} + +func TestEdgeGetCapabilityInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad-uuid"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_get_connection/tool.go b/pkg/tools/edge_get_connection/tool.go new file mode 100644 index 0000000..1b00a1f --- /dev/null +++ b/pkg/tools/edge_get_connection/tool.go @@ -0,0 +1,45 @@ +package edge_get_connection + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + ConnectionId string `json:"connectionId" jsonschema:"the UUID of the connection to retrieve"` +} + +type Output struct { + Connection *clients.EdgeConnection `json:"connection,omitempty" jsonschema:"the connection details"` + Found bool `json:"found" jsonschema:"whether the connection was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_get_connection", + Description: "Gets an Edge connection by its UUID. Returns the connection's configuration, type, edge site, and parameters.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("connectionId", input.ConnectionId); err != nil { + return Output{}, err + } + conn, err := clients.GetConnection(ctx, collibraClient, input.ConnectionId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get connection: %s", err.Error())}, nil + } + return Output{Connection: conn, Found: true}, nil + } +} diff --git a/pkg/tools/edge_get_connection/tool_test.go b/pkg/tools/edge_get_connection/tool_test.go new file mode 100644 index 0000000..cf0ebc7 --- /dev/null +++ b/pkg/tools/edge_get_connection/tool_test.go @@ -0,0 +1,64 @@ +package edge_get_connection_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_get_connection" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeGetConnection(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.EdgeConnection) { + return http.StatusOK, clients.EdgeConnection{Id: "00000000-0000-0000-0000-000000000001", Name: "Test Conn"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + ConnectionId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Connection.Name != "Test Conn" { + t.Fatalf("expected name Test Conn, got %s", output.Connection.Name) + } +} + +func TestEdgeGetConnectionNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections/00000000-0000-0000-0000-000000000002", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + ConnectionId: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestEdgeGetConnectionInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ConnectionId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_get_job_status/tool.go b/pkg/tools/edge_get_job_status/tool.go new file mode 100644 index 0000000..9bf6456 --- /dev/null +++ b/pkg/tools/edge_get_job_status/tool.go @@ -0,0 +1,49 @@ +// Package edge_get_job_status implements the edge_get_job_status MCP tool: polls the status of +// an Edge job, e.g. one started by test_connection or start_ingestion. +package edge_get_job_status + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + JobID string `json:"jobId" jsonschema:"UUID of the job to check, as returned by test_connection or start_ingestion."` +} + +type Output struct { + Status string `json:"status,omitempty" jsonschema:"The job's current status: UNKNOWN, SUBMITTED, DOWNLOADING, SCHEDULED, RUNNING, SUCCEEDED, FAILED, CANCELLED, CANCELLING, CAPABILITY_PROGRESS, CAPABILITY_SUCCEEDED, or CAPABILITY_FAILED. SUCCEEDED/FAILED/CANCELLED/CAPABILITY_SUCCEEDED/CAPABILITY_FAILED are terminal."` + Message string `json:"message,omitempty" jsonschema:"Status message from the job."` + LastUpdatedDateTime string `json:"lastUpdatedDateTime,omitempty" jsonschema:"When the status was last updated."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_get_job_status", + Title: "Get Edge Job Status", + Description: "Gets the current status of an Edge job, e.g. one started by test_connection or start_ingestion.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("jobId", input.JobID); err != nil { + return Output{}, err + } + + status, err := clients.GetJobStatusLog(ctx, collibraClient, input.JobID) + if err != nil { + return Output{}, err + } + + return Output{Status: status.Status, Message: status.Message, LastUpdatedDateTime: status.LastUpdatedDateTime}, nil + } +} diff --git a/pkg/tools/edge_get_job_status/tool_test.go b/pkg/tools/edge_get_job_status/tool_test.go new file mode 100644 index 0000000..c13b73e --- /dev/null +++ b/pkg/tools/edge_get_job_status/tool_test.go @@ -0,0 +1,46 @@ +package edge_get_job_status_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_get_job_status" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestGetJobStatus(t *testing.T) { + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/jobs/"+jobID.String()+"/statusLog", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobStatusLog) { + return http.StatusOK, clients.JobStatusLog{ + JobID: jobID.String(), + Status: "CAPABILITY_SUCCEEDED", + Message: "done", + LastUpdatedDateTime: "2026-07-04T10:00:00.000Z", + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: jobID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Status != "CAPABILITY_SUCCEEDED" { + t.Fatalf("unexpected status: %s", output.Status) + } +} + +func TestGetJobStatus_InvalidJobID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid jobId") + } +} diff --git a/pkg/tools/edge_get_job_status_history/tool.go b/pkg/tools/edge_get_job_status_history/tool.go new file mode 100644 index 0000000..d87c548 --- /dev/null +++ b/pkg/tools/edge_get_job_status_history/tool.go @@ -0,0 +1,48 @@ +package edge_get_job_status_history + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the Edge job to get status history for"` +} + +type Output struct { + History []clients.JobStatusLog `json:"history" jsonschema:"all job status updates in LIFO order (most recent first)"` + Count int `json:"count" jsonschema:"number of status entries returned"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_get_job_status_history", + Description: "Gets all status updates for an Edge job in reverse chronological order (most recent first). Useful for diagnosing job failures.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + history, err := clients.GetEdgeJobStatusHistory(ctx, collibraClient, input.Id) + if err != nil { + return Output{History: []clients.JobStatusLog{}, Error: fmt.Sprintf("failed to get job status history: %s", err.Error())}, nil + } + if history == nil { + history = []clients.JobStatusLog{} + } + return Output{History: history, Count: len(history)}, nil + } +} diff --git a/pkg/tools/edge_get_job_status_history/tool_test.go b/pkg/tools/edge_get_job_status_history/tool_test.go new file mode 100644 index 0000000..ca4bae8 --- /dev/null +++ b/pkg/tools/edge_get_job_status_history/tool_test.go @@ -0,0 +1,70 @@ +package edge_get_job_status_history_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_get_job_status_history" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeGetJobStatusHistory(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/jobs/00000000-0000-0000-0000-000000000001/statusLogHistory", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.JobStatusLog) { + return http.StatusOK, []clients.JobStatusLog{ + {Status: "SUCCEEDED", Message: "Done"}, + {Status: "RUNNING", Message: "In progress"}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + if len(output.History) != 2 { + t.Fatalf("expected 2 entries, got %d", len(output.History)) + } + if output.History[0].Status != "SUCCEEDED" { + t.Fatalf("expected SUCCEEDED first, got %s", output.History[0].Status) + } +} + +func TestEdgeGetJobStatusHistoryAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/jobs/00000000-0000-0000-0000-000000000001/statusLogHistory", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error") + } +} + +func TestEdgeGetJobStatusHistoryInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_list_capabilities/tool.go b/pkg/tools/edge_list_capabilities/tool.go new file mode 100644 index 0000000..b958532 --- /dev/null +++ b/pkg/tools/edge_list_capabilities/tool.go @@ -0,0 +1,74 @@ +package edge_list_capabilities + +import ( + "context" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + NameContains string `json:"nameContains,omitempty" jsonschema:"optional substring to filter capability names (case-insensitive)"` + TypeId string `json:"typeId,omitempty" jsonschema:"optional capability type ID to filter by, e.g. databricks-edge-capability"` + Limit int `json:"limit,omitempty" jsonschema:"optional maximum number of results to return"` +} + +type Output struct { + Capabilities []clients.EdgeCapability `json:"capabilities" jsonschema:"list of Edge capabilities"` + Count int `json:"count" jsonschema:"number of capabilities returned"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_list_capabilities", + Description: "Gets Edge capabilities with optional filtering by name (substring) and/or type ID. Use nameContains and typeId to avoid fetching the full list, which can be very large.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + caps, err := clients.ListCapabilities(ctx, collibraClient) + if err != nil { + return Output{Capabilities: []clients.EdgeCapability{}, Error: err.Error()}, nil + } + if caps == nil { + caps = []clients.EdgeCapability{} + } + + filtered := filter(caps, input) + return Output{Capabilities: filtered, Count: len(filtered)}, nil + } +} + +func filter(caps []clients.EdgeCapability, input Input) []clients.EdgeCapability { + result := make([]clients.EdgeCapability, 0, len(caps)) + nameFilter := strings.ToLower(input.NameContains) + typeFilter := strings.ToLower(input.TypeId) + + for _, c := range caps { + if nameFilter != "" && !strings.Contains(strings.ToLower(c.Name), nameFilter) { + continue + } + if typeFilter != "" { + typeId := "" + if c.Type != nil { + typeId = strings.ToLower(c.Type.Id) + } + if typeId != typeFilter { + continue + } + } + result = append(result, c) + if input.Limit > 0 && len(result) >= input.Limit { + break + } + } + return result +} diff --git a/pkg/tools/edge_list_capabilities/tool_test.go b/pkg/tools/edge_list_capabilities/tool_test.go new file mode 100644 index 0000000..668b19f --- /dev/null +++ b/pkg/tools/edge_list_capabilities/tool_test.go @@ -0,0 +1,144 @@ +package edge_list_capabilities_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_list_capabilities" + "github.com/collibra/chip/pkg/tools/testutil" +) + +var testCapabilities = []clients.EdgeCapability{ + {Id: "cap-1", Name: "vb-databricks-prod", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "cap-2", Name: "vb-databricks-staging", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "cap-3", Name: "finance-dataplex", Type: &clients.EdgeCapabilityType{Id: "dataplex-edge-capability"}}, + {Id: "cap-4", Name: "sales-databricks", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, +} + +func newServer(t *testing.T) *httptest.Server { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, testCapabilities + })) + return httptest.NewServer(handler) +} + +func TestEdgeListCapabilities(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + if output.Count != 4 { + t.Fatalf("expected 4 capabilities, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesFilterByName(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{NameContains: "vb"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 2 { + t.Fatalf("expected 2 capabilities, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesFilterByType(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{TypeId: "databricks-edge-capability"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 3 { + t.Fatalf("expected 3 databricks capabilities, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesFilterByNameAndType(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + NameContains: "vb", + TypeId: "databricks-edge-capability", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 2 { + t.Fatalf("expected 2 capabilities, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesFilterByNameCaseInsensitive(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{NameContains: "VB"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 2 { + t.Fatalf("expected 2 capabilities (case-insensitive), got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesLimit(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Limit: 2}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 2 { + t.Fatalf("expected 2 capabilities with limit, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "internal server error" + })) + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error to be set") + } +} + +func TestEdgeListCapabilitiesEmpty(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{} + })) + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 0 { + t.Fatalf("expected count 0, got %d", output.Count) + } +} diff --git a/pkg/tools/edge_list_capability_types/tool.go b/pkg/tools/edge_list_capability_types/tool.go new file mode 100644 index 0000000..5798e72 --- /dev/null +++ b/pkg/tools/edge_list_capability_types/tool.go @@ -0,0 +1,93 @@ +// Package edge_list_capability_types implements the edge_list_capability_types MCP tool: a +// read-only discovery tool for finding valid typeId values (and their manifests) for +// edge_create_capability and edge_create_connection on a given edge site. +package edge_list_capability_types + +import ( + "context" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site to list available capability and connection types for. Use the edge_list_sites tool to discover available sites."` + Query string `json:"query,omitempty" jsonschema:"Optional. Case-insensitive substring filter on type id (e.g. 'jdbc', 'snowflake'). Without a query, all type ids are returned WITHOUT their manifest (an edge site can have 80+ types; manifests are large) — call again with a query matching the type you want to see its full manifest with expected parameters."` +} + +type Output struct { + CapabilityTypes []clients.CapabilityType `json:"capabilityTypes" jsonschema:"Capability types available on this edge site (e.g. 'jdbc-ingestion'). Manifest is only populated when query narrows the results — see query's description."` + ConnectionTypes []clients.ConnectionType `json:"connectionTypes" jsonschema:"Connection types available on this edge site (e.g. 'Generic' or a vendor-specific type). Manifest is only populated when query narrows the results — see query's description."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_list_capability_types", + Title: "List Capability and Connection Types", + Description: "Lists the capability and connection types available on an edge site. Without query, returns just the ids (an edge site can have 80+ types, each with a large manifest). Pass query to filter to matching types and get their full manifest describing expected parameters for edge_create_capability/edge_create_connection.", + Handler: handler(collibraClient), + Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("edgeSiteId", input.EdgeSiteID); err != nil { + return Output{}, err + } + + capabilityTypes, err := clients.GetCapabilityTypes(ctx, collibraClient, input.EdgeSiteID) + if err != nil { + return Output{}, err + } + + connectionTypes, err := clients.GetConnectionTypes(ctx, collibraClient, input.EdgeSiteID) + if err != nil { + return Output{}, err + } + + query := strings.ToLower(strings.TrimSpace(input.Query)) + if query == "" { + return Output{ + CapabilityTypes: stripManifests(capabilityTypes), + ConnectionTypes: stripManifestsConn(connectionTypes), + }, nil + } + + matchedCapabilities := []clients.CapabilityType{} + for _, c := range capabilityTypes { + if strings.Contains(strings.ToLower(c.ID), query) { + matchedCapabilities = append(matchedCapabilities, c) + } + } + matchedConnections := []clients.ConnectionType{} + for _, c := range connectionTypes { + if strings.Contains(strings.ToLower(c.ID), query) { + matchedConnections = append(matchedConnections, c) + } + } + + return Output{CapabilityTypes: matchedCapabilities, ConnectionTypes: matchedConnections}, nil + } +} + +func stripManifests(types []clients.CapabilityType) []clients.CapabilityType { + out := make([]clients.CapabilityType, len(types)) + for i, t := range types { + out[i] = clients.CapabilityType{ID: t.ID} + } + return out +} + +func stripManifestsConn(types []clients.ConnectionType) []clients.ConnectionType { + out := make([]clients.ConnectionType, len(types)) + for i, t := range types { + out[i] = clients.ConnectionType{ID: t.ID} + } + return out +} diff --git a/pkg/tools/edge_list_capability_types/tool_test.go b/pkg/tools/edge_list_capability_types/tool_test.go new file mode 100644 index 0000000..9f93a28 --- /dev/null +++ b/pkg/tools/edge_list_capability_types/tool_test.go @@ -0,0 +1,115 @@ +package edge_list_capability_types_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_list_capability_types" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestListCapabilityTypes(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/capabilityTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.CapabilityType) { + return http.StatusOK, []clients.CapabilityType{{ID: "jdbc-ingestion"}} + })) + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/connectionTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.ConnectionType) { + return http.StatusOK, []clients.ConnectionType{{ID: "Generic"}} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: siteID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.CapabilityTypes) != 1 || output.CapabilityTypes[0].ID != "jdbc-ingestion" { + t.Fatalf("unexpected capability types: %+v", output.CapabilityTypes) + } + if len(output.ConnectionTypes) != 1 || output.ConnectionTypes[0].ID != "Generic" { + t.Fatalf("unexpected connection types: %+v", output.ConnectionTypes) + } +} + +func TestListCapabilityTypes_NoQueryStripsManifests(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/capabilityTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.CapabilityType) { + return http.StatusOK, []clients.CapabilityType{ + {ID: "jdbc-ingestion", Manifest: map[string]any{"huge": "manifest"}}, + {ID: "snowflake-synchronization", Manifest: map[string]any{"huge": "manifest"}}, + } + })) + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/connectionTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.ConnectionType) { + return http.StatusOK, []clients.ConnectionType{{ID: "Generic", Manifest: map[string]any{"huge": "manifest"}}} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: siteID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.CapabilityTypes) != 2 { + t.Fatalf("expected 2 capability types, got: %+v", output.CapabilityTypes) + } + for _, c := range output.CapabilityTypes { + if c.Manifest != nil { + t.Fatalf("expected manifest to be stripped without a query, got: %+v", c) + } + } +} + +func TestListCapabilityTypes_QueryFiltersAndKeepsManifest(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/capabilityTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.CapabilityType) { + return http.StatusOK, []clients.CapabilityType{ + {ID: "jdbc-ingestion", Manifest: map[string]any{"connection": "type"}}, + {ID: "snowflake-synchronization", Manifest: map[string]any{"other": "type"}}, + } + })) + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/connectionTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.ConnectionType) { + return http.StatusOK, []clients.ConnectionType{{ID: "Generic"}} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: siteID.String(), Query: "jdbc"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.CapabilityTypes) != 1 || output.CapabilityTypes[0].ID != "jdbc-ingestion" { + t.Fatalf("expected only jdbc-ingestion to match, got: %+v", output.CapabilityTypes) + } + if output.CapabilityTypes[0].Manifest == nil { + t.Fatalf("expected manifest to be kept when query filters results") + } + if len(output.ConnectionTypes) != 0 { + t.Fatalf("expected no connection types to match 'jdbc', got: %+v", output.ConnectionTypes) + } + if output.ConnectionTypes == nil { + t.Fatalf("expected an empty slice, not nil — a nil slice marshals to JSON null, which fails the tool's array-typed output schema") + } +} + +func TestListCapabilityTypes_InvalidEdgeSiteID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid edgeSiteId") + } +} diff --git a/pkg/tools/edge_list_connections/tool.go b/pkg/tools/edge_list_connections/tool.go new file mode 100644 index 0000000..a241aa0 --- /dev/null +++ b/pkg/tools/edge_list_connections/tool.go @@ -0,0 +1,41 @@ +package edge_list_connections + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct{} + +type Output struct { + Connections []clients.EdgeConnection `json:"connections" jsonschema:"list of Edge connections"` + Count int `json:"count" jsonschema:"number of connections returned"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_list_connections", + Description: "Gets all available Edge connections. Returns the full list of connections configured on the Collibra Edge management service.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + conns, err := clients.ListConnections(ctx, collibraClient) + if err != nil { + return Output{Connections: []clients.EdgeConnection{}, Error: err.Error()}, nil + } + if conns == nil { + conns = []clients.EdgeConnection{} + } + return Output{Connections: conns, Count: len(conns)}, nil + } +} diff --git a/pkg/tools/edge_list_connections/tool_test.go b/pkg/tools/edge_list_connections/tool_test.go new file mode 100644 index 0000000..3ffc69d --- /dev/null +++ b/pkg/tools/edge_list_connections/tool_test.go @@ -0,0 +1,55 @@ +package edge_list_connections_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_list_connections" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeListConnections(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeConnection) { + return http.StatusOK, []clients.EdgeConnection{ + {Id: "conn-1", Name: "Databricks Conn"}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + if len(output.Connections) != 1 || output.Connections[0].Id != "conn-1" { + t.Fatalf("expected conn-1, got %+v", output.Connections) + } + if output.Count != 1 { + t.Fatalf("expected count 1, got %d", output.Count) + } +} + +func TestEdgeListConnectionsAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "error" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error") + } +} diff --git a/pkg/tools/edge_list_sites/tool.go b/pkg/tools/edge_list_sites/tool.go new file mode 100644 index 0000000..c27827c --- /dev/null +++ b/pkg/tools/edge_list_sites/tool.go @@ -0,0 +1,40 @@ +// Package edge_list_sites implements the edge_list_sites MCP tool: a read-only +// discovery tool for finding the edgeSiteId needed by edge_create_connection and +// edge_create_capability. +package edge_list_sites + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct{} + +type Output struct { + Sites []clients.EdgeSite `json:"sites" jsonschema:"The list of available edge sites."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_list_sites", + Title: "List Edge Sites", + Description: "Lists available Edge sites, including their id, name, and status. Use this to find the edgeSiteId needed by edge_create_connection and edge_create_capability.", + Handler: handler(collibraClient), + Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, _ Input) (Output, error) { + sites, err := clients.GetEdgeSites(ctx, collibraClient) + if err != nil { + return Output{}, err + } + return Output{Sites: sites}, nil + } +} diff --git a/pkg/tools/edge_list_sites/tool_test.go b/pkg/tools/edge_list_sites/tool_test.go new file mode 100644 index 0000000..2362c25 --- /dev/null +++ b/pkg/tools/edge_list_sites/tool_test.go @@ -0,0 +1,38 @@ +package edge_list_sites_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_list_sites" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestListEdgeSites(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/sites", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeSite) { + return http.StatusOK, []clients.EdgeSite{ + {ID: siteID.String(), Name: "local-dev", Status: "HEALTHY"}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.Sites) != 1 { + t.Fatalf("expected 1 site, got: %d", len(output.Sites)) + } + if output.Sites[0].Name != "local-dev" { + t.Fatalf("expected name 'local-dev', got: '%s'", output.Sites[0].Name) + } +} diff --git a/pkg/tools/edge_run_capability/tool.go b/pkg/tools/edge_run_capability/tool.go new file mode 100644 index 0000000..2c40162 --- /dev/null +++ b/pkg/tools/edge_run_capability/tool.go @@ -0,0 +1,57 @@ +package edge_run_capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the capability to run"` + JobId string `json:"jobId,omitempty" jsonschema:"optional UUID for tracking the job execution"` + InFastNamespace bool `json:"inFastNamespace,omitempty" jsonschema:"if true, runs in the fast (priority) namespace"` + Parameters map[string]any `json:"parameters,omitempty" jsonschema:"optional run-time parameters for this execution only"` + WorkflowName string `json:"workflowName,omitempty" jsonschema:"optional workflow name; defaults to workflow.yaml inside the capability package"` +} + +type Output struct { + JobId string `json:"jobId,omitempty" jsonschema:"UUID of the job submitted to Edge"` + Success bool `json:"success" jsonschema:"whether the run was submitted successfully"` + Error string `json:"error,omitempty" jsonschema:"error message if submission failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_run_capability", + Description: "Runs an Edge capability immediately. Returns the job UUID that can be used to track execution via edge_get_job_status.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + if err := validation.UUIDOptional("jobId", input.JobId); err != nil { + return Output{}, err + } + jobId, err := clients.RunCapability(ctx, collibraClient, input.Id, clients.CapabilityRunRequest{ + JobId: input.JobId, + InFastNamespace: input.InFastNamespace, + Parameters: input.Parameters, + WorkflowName: input.WorkflowName, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to run capability: %s", err.Error())}, nil + } + return Output{JobId: jobId, Success: true}, nil + } +} diff --git a/pkg/tools/edge_run_capability/tool_test.go b/pkg/tools/edge_run_capability/tool_test.go new file mode 100644 index 0000000..5e94f8b --- /dev/null +++ b/pkg/tools/edge_run_capability/tool_test.go @@ -0,0 +1,66 @@ +package edge_run_capability_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/edge_run_capability" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeRunCapability(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000001/run", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusCreated, "00000000-0000-0000-0000-000000000099" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } + if output.JobId != "00000000-0000-0000-0000-000000000099" { + t.Fatalf("expected job id 00000000-0000-0000-0000-000000000099, got: %s", output.JobId) + } +} + +func TestEdgeRunCapabilityAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000001/run", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "capability not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } + if output.Error == "" { + t.Fatal("expected error message") + } +} + +func TestEdgeRunCapabilityInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/find_domain_types/tool.go b/pkg/tools/find_domain_types/tool.go new file mode 100644 index 0000000..1c987aa --- /dev/null +++ b/pkg/tools/find_domain_types/tool.go @@ -0,0 +1,70 @@ +// Package find_domain_types implements the find_domain_types MCP tool: resolves a +// domain type's name (e.g. "Physical Data Dictionary") to its UUID, for +// create_domain's typeId — there is no other discovery path for this, since domain +// types are not returned by edge_list_capability_types or any Edge API. +package find_domain_types + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name,omitempty" jsonschema:"Optional. Matches case-insensitively as a substring against the domain type name (e.g. 'Physical Data Dictionary', 'Technology Asset'). Omit to list all domain types."` + Limit int `json:"limit,omitempty" jsonschema:"Optional. Maximum number of results to return. Default: 50."` + Offset int `json:"offset,omitempty" jsonschema:"Optional. Index of first result (pagination offset). Default: 0."` +} + +type Output struct { + Total int64 `json:"total" jsonschema:"The total number of domain types matching the search criteria."` + Offset int64 `json:"offset" jsonschema:"The offset for the results."` + Limit int64 `json:"limit" jsonschema:"The maximum number of results returned."` + DomainTypes []DomainType `json:"domainTypes" jsonschema:"The matching domain types."` +} + +type DomainType struct { + ID string `json:"id" jsonschema:"The domain type's UUID — use this for create_domain's typeId."` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + PublicID string `json:"publicId,omitempty"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "find_domain_types", + Title: "Find Domain Types", + Description: "Finds DGC domain types by name (substring, case-insensitive), returning their UUIDs. Use this to resolve a domain type name (e.g. 'Physical Data Dictionary', needed for jdbc-ingestion databases) to a typeId for create_domain, instead of needing to already know the UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + limit := input.Limit + if limit == 0 { + limit = 50 + } + + response, err := clients.FindDomainTypes(ctx, collibraClient, clients.FindDomainTypesQueryParams{ + Name: input.Name, + Limit: limit, + Offset: input.Offset, + }) + if err != nil { + return Output{}, err + } + + domainTypes := make([]DomainType, len(response.Results)) + for i, d := range response.Results { + domainTypes[i] = DomainType{ID: d.ID, Name: d.Name, Description: d.Description, PublicID: d.PublicID} + } + + return Output{Total: response.Total, Offset: response.Offset, Limit: response.Limit, DomainTypes: domainTypes}, nil + } +} diff --git a/pkg/tools/find_domain_types/tool_test.go b/pkg/tools/find_domain_types/tool_test.go new file mode 100644 index 0000000..44d13d3 --- /dev/null +++ b/pkg/tools/find_domain_types/tool_test.go @@ -0,0 +1,39 @@ +package find_domain_types_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/find_domain_types" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindDomainTypes(t *testing.T) { + typeID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/domainTypes", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("limit") != "50" { + t.Fatalf("expected default limit 50, got: %s", r.URL.Query().Get("limit")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":1,"offset":0,"limit":50,"results":[{"id":"` + typeID.String() + `","name":"Physical Data Dictionary"}]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Name: "Physical"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Total != 1 || len(output.DomainTypes) != 1 { + t.Fatalf("unexpected output: %+v", output) + } + if output.DomainTypes[0].ID != typeID.String() { + t.Fatalf("expected id %s, got %s", typeID.String(), output.DomainTypes[0].ID) + } +} diff --git a/pkg/tools/find_users/tool.go b/pkg/tools/find_users/tool.go new file mode 100644 index 0000000..9edbee0 --- /dev/null +++ b/pkg/tools/find_users/tool.go @@ -0,0 +1,80 @@ +// Package find_users implements the find_users MCP tool: resolves a user's name +// (or partial name) to their UUID, e.g. to fill in register_database's ownerIds +// without the caller needing to already know the id. +package find_users + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name,omitempty" jsonschema:"Optional. Matches case-insensitively as a substring against username, first name, last name, and first+last name combinations (e.g. 'admin', 'jane', 'jane doe'). Omit to list users."` + IncludeDisabled bool `json:"includeDisabled,omitempty" jsonschema:"Optional. If true, includes disabled user accounts. Defaults to false (enabled users only)."` + Limit int `json:"limit,omitempty" jsonschema:"Optional. Maximum number of results to return. Default: 20."` + Offset int `json:"offset,omitempty" jsonschema:"Optional. Index of first result (pagination offset). Default: 0."` +} + +type Output struct { + Total int64 `json:"total" jsonschema:"The total number of users matching the search criteria."` + Offset int64 `json:"offset" jsonschema:"The offset for the results."` + Limit int64 `json:"limit" jsonschema:"The maximum number of results returned."` + Users []User `json:"users" jsonschema:"The matching users."` +} + +type User struct { + ID string `json:"id" jsonschema:"The user's UUID — use this for ownerIds, createdBy, etc."` + UserName string `json:"userName,omitempty" jsonschema:"The user's username."` + FirstName string `json:"firstName,omitempty"` + LastName string `json:"lastName,omitempty"` + EmailAddress string `json:"emailAddress,omitempty"` + Enabled bool `json:"enabled" jsonschema:"Whether the user account is enabled."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "find_users", + Title: "Find Users", + Description: "Finds DGC users by name (substring, case-insensitive), returning their UUIDs. Use this to resolve a name to an id for register_database's ownerIds or similar owner/assignee fields, instead of needing to already know the UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + limit := input.Limit + if limit == 0 { + limit = 20 + } + + response, err := clients.FindUsers(ctx, collibraClient, clients.FindUsersQueryParams{ + Name: input.Name, + Limit: limit, + Offset: input.Offset, + IncludeDisabled: input.IncludeDisabled, + }) + if err != nil { + return Output{}, err + } + + users := make([]User, len(response.Results)) + for i, u := range response.Results { + users[i] = User{ + ID: u.ID, + UserName: u.UserName, + FirstName: u.FirstName, + LastName: u.LastName, + EmailAddress: u.EmailAddress, + Enabled: u.Enabled, + } + } + + return Output{Total: response.Total, Offset: response.Offset, Limit: response.Limit, Users: users}, nil + } +} diff --git a/pkg/tools/find_users/tool_test.go b/pkg/tools/find_users/tool_test.go new file mode 100644 index 0000000..9ed377f --- /dev/null +++ b/pkg/tools/find_users/tool_test.go @@ -0,0 +1,65 @@ +package find_users_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/find_users" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindUsers(t *testing.T) { + adminID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/users", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("name") != "Admin" { + t.Fatalf("unexpected name query param: %s", r.URL.Query().Get("name")) + } + if r.URL.Query().Get("limit") != "20" { + t.Fatalf("expected default limit 20, got: %s", r.URL.Query().Get("limit")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":1,"offset":0,"limit":20,"results":[{"id":"` + adminID.String() + `","userName":"Admin","emailAddress":"admin@example.com","enabled":true}]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Name: "Admin"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Total != 1 || len(output.Users) != 1 { + t.Fatalf("unexpected output: %+v", output) + } + if output.Users[0].ID != adminID.String() { + t.Fatalf("expected id %s, got %s", adminID.String(), output.Users[0].ID) + } +} + +func TestFindUsers_CustomLimit(t *testing.T) { + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/users", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("limit") != "5" { + t.Fatalf("expected limit 5, got: %s", r.URL.Query().Get("limit")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":0,"offset":0,"limit":5,"results":[]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Limit: 5}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.Users) != 0 { + t.Fatalf("expected no users, got: %+v", output.Users) + } +} diff --git a/pkg/tools/get_data_source_setup_guide/tool.go b/pkg/tools/get_data_source_setup_guide/tool.go new file mode 100644 index 0000000..65505c9 --- /dev/null +++ b/pkg/tools/get_data_source_setup_guide/tool.go @@ -0,0 +1,124 @@ +// Package get_data_source_setup_guide implements the get_data_source_setup_guide MCP +// tool: looks up driver class, JDBC connection string format, and per-auth-method +// connection properties for any of Collibra's ~68 supported data sources, so +// edge_create_connection can be filled in correctly without the agent (or the user) +// needing to already know a given vendor's JDBC driver properties. +// +// This is a documented gap, not a guaranteed integration: the underlying data comes +// from scraping Collibra's public product documentation site (see the package doc +// on clients.GetConnectionProperties for the full explanation and how to re-derive +// it if it breaks). There is no published API for this — Collibra could restructure +// their docs site at any time without notice. Treat a failure from this tool as "ask +// the user for the connection properties directly," not a bug to retry around. +package get_data_source_setup_guide + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + DataSource string `json:"dataSource" jsonschema:"The data source to look up (e.g. 'Snowflake', 'PostgreSQL', 'Amazon Redshift'). Matched case-insensitively against Collibra's ~68 supported data sources; if ambiguous or not found, the response lists candidates to retry with."` +} + +type Output struct { + DataSourceID string `json:"dataSourceId,omitempty" jsonschema:"The matched data source's internal id (slug)."` + DataSourceName string `json:"dataSourceName,omitempty" jsonschema:"The matched data source's display name."` + DriverClassName string `json:"driverClassName,omitempty" jsonschema:"The JDBC driver class name for edge_create_connection's 'driver-class' parameter."` + ConnectionStringFormat string `json:"connectionStringFormat,omitempty" jsonschema:"The JDBC connection string format for edge_create_connection's 'connection-string' parameter, with placeholders (e.g. 'jdbc:snowflake://.snowflakecomputing.com')."` + MarketplaceURL string `json:"marketplaceUrl,omitempty" jsonschema:"Collibra Marketplace listing URL for this data source's JDBC driver, if the driver needs to be downloaded before uploading via edge_create_connection's driverJarUrl/driverJarFilename or the upload_file tool."` + Prerequisites string `json:"prerequisites,omitempty" jsonschema:"Documented prerequisites before creating this connection."` + AuthMethods []clients.AuthMethodProperties `json:"authMethods,omitempty" jsonschema:"Documented connection properties, grouped by supported authentication method. Pick the method matching what credentials the user has, then pass its properties as edge_create_connection's additionalProperties (name/type/value/secret per entry — secret=true when the property's Value Type here is 'Secret'). A 'File' type property's value must first be uploaded via upload_file or edge_create_connection's driverJarUrl, then referenced by the returned artifact URI."` + Matches []clients.DataSourceSlug `json:"matches,omitempty" jsonschema:"Candidate data sources when the lookup was ambiguous or not found. Retry with one of these exact ids."` + Success bool `json:"success" jsonschema:"Whether at least some setup information was found."` + Error string `json:"error,omitempty" jsonschema:"Details if part or all of the lookup failed. A failure here does not mean the data source is unsupported by Edge — it means this best-effort documentation lookup didn't find it; ask the user for the connection properties directly."` +} + +func NewTool(_ *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_data_source_setup_guide", + Title: "Get Data Source Connection Setup Guide", + Description: "Looks up driver class, JDBC connection string format, and per-auth-method connection properties for a data source, from Collibra's public documentation (best-effort scrape, not a stable API — see tool source for details). Use before edge_create_connection when the required parameters for a data source (e.g. Snowflake's Role/Warehouse/private key) aren't already known.", + Handler: handler(), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +// DocsClient hits Collibra's public documentation site directly, never the tenant's +// Collibra instance — it must not use the injected, tenant-scoped Collibra client. +// Overridable in tests. +var DocsClient = http.DefaultClient + +func handler() chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if strings.TrimSpace(input.DataSource) == "" { + return Output{}, fmt.Errorf("dataSource is required") + } + + sources, err := clients.ListDataSources(ctx, DocsClient) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to list data sources: %s", err.Error())}, nil + } + + matches := matchDataSources(sources, input.DataSource) + if len(matches) == 0 { + return Output{Success: false, Error: fmt.Sprintf("no data source found matching %q", input.DataSource), Matches: sources}, nil + } + if len(matches) > 1 { + return Output{Success: false, Error: fmt.Sprintf("multiple data sources matched %q; retry with one exact id", input.DataSource), Matches: matches}, nil + } + + slug := matches[0] + output := Output{DataSourceID: slug.ID, DataSourceName: slug.Name} + + var errs []string + + if info, err := clients.GetDataSourceInfo(ctx, DocsClient, slug.ID); err != nil { + errs = append(errs, fmt.Sprintf("driver/connection-string info unavailable: %s", err.Error())) + } else { + output.DriverClassName = info.DriverClassName + output.ConnectionStringFormat = info.ConnectionStringFormat + output.MarketplaceURL = info.MarketplaceURL + output.Prerequisites = info.Prerequisites + } + + if methods, err := clients.GetConnectionProperties(ctx, DocsClient, slug.ID); err != nil { + errs = append(errs, fmt.Sprintf("connection properties unavailable: %s", err.Error())) + } else { + output.AuthMethods = methods + } + + output.Success = output.DriverClassName != "" || len(output.AuthMethods) > 0 + output.Error = strings.Join(errs, "; ") + return output, nil + } +} + +// matchDataSources resolves a free-text query against the known data source list: +// exact id/name match wins outright; otherwise falls back to substring matches. +func matchDataSources(sources []clients.DataSourceSlug, query string) []clients.DataSourceSlug { + q := strings.ToLower(strings.TrimSpace(query)) + + var exact, partial []clients.DataSourceSlug + for _, s := range sources { + id, name := strings.ToLower(s.ID), strings.ToLower(s.Name) + switch { + case id == q || name == q: + exact = append(exact, s) + case strings.Contains(id, q) || strings.Contains(name, q): + partial = append(partial, s) + } + } + + if len(exact) > 0 { + return exact + } + return partial +} diff --git a/pkg/tools/get_data_source_setup_guide/tool_test.go b/pkg/tools/get_data_source_setup_guide/tool_test.go new file mode 100644 index 0000000..0362e3f --- /dev/null +++ b/pkg/tools/get_data_source_setup_guide/tool_test.go @@ -0,0 +1,139 @@ +package get_data_source_setup_guide_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/get_data_source_setup_guide" + "github.com/collibra/chip/pkg/tools/testutil" +) + +const selectorFixture = ` +
      +
    • Snowflake
    • +
    • PostgreSQL
    • +
    • PostgreSQL Classic
    • +
    +` + +const dataSourceInfoFixture = ` +

    The JDBC connection string:

    +

    jdbc:snowflake://<accountname>.snowflakecomputing.com

    +

    The JDBC driver class name:

    +
    net.snowflake.client.jdbc.SnowflakeDriver
    +` + +const connectionPropertiesFixture = ` +
    + +

    Required?

    Name

    Type

    Value Type

    Value

    YesPasswordTextSecretThe password.
    +
    +` + +func newTestServer(t *testing.T) *http.Client { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/_all-data-sources/selector/selector.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(selectorFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/snowflake/data-source-information.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(dataSourceInfoFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/snowflake/minimum-connection-properties-edge.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(connectionPropertiesFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/postgresql/data-source-information.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(dataSourceInfoFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/postgresql/minimum-connection-properties-edge.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(connectionPropertiesFixture)) + }) + return testutil.NewClient(httptest.NewServer(mux)) +} + +func TestGetDataSourceSetupGuide_Success(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "Snowflake"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.DataSourceID != "snowflake" { + t.Fatalf("unexpected data source id: %s", output.DataSourceID) + } + if output.DriverClassName != "net.snowflake.client.jdbc.SnowflakeDriver" { + t.Fatalf("unexpected driver class: %s", output.DriverClassName) + } + if len(output.AuthMethods) != 1 || len(output.AuthMethods[0].Properties) != 1 { + t.Fatalf("unexpected auth methods: %+v", output.AuthMethods) + } +} + +func TestGetDataSourceSetupGuide_CaseInsensitiveMatch(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "snowflake"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestGetDataSourceSetupGuide_Ambiguous(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "postgresql"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + // "postgresql" exact-matches one id but is also a substring of "postgresql-classic"; + // exact match should win outright rather than reporting ambiguity. + if !output.Success || output.DataSourceID != "postgresql" { + t.Fatalf("expected exact id match to win, got: %+v", output) + } +} + +func TestGetDataSourceSetupGuide_TrulyAmbiguous(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "postgres"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Success { + t.Fatalf("expected failure due to ambiguity") + } + if len(output.Matches) != 2 { + t.Fatalf("expected 2 candidate matches, got: %+v", output.Matches) + } +} + +func TestGetDataSourceSetupGuide_NotFound(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "sap-hana"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Success { + t.Fatalf("expected failure for unknown data source") + } + if len(output.Matches) != 3 { + t.Fatalf("expected all 3 known sources returned as candidates, got: %+v", output.Matches) + } +} + +func TestGetDataSourceSetupGuide_EmptyInput(t *testing.T) { + tools.DocsClient = newTestServer(t) + + _, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: ""}) + if err == nil { + t.Fatalf("expected an error for empty dataSource") + } +} diff --git a/pkg/tools/get_job_status/tool.go b/pkg/tools/get_job_status/tool.go new file mode 100644 index 0000000..3d36208 --- /dev/null +++ b/pkg/tools/get_job_status/tool.go @@ -0,0 +1,65 @@ +// Package get_job_status implements the get_job_status MCP tool: +// polls the status of a DGC job, e.g. the one returned by start_ingestion. +// This is distinct from edge_get_job_status, which polls Edge-site jobs (capability runs, +// connection tests) — the two job id spaces are not interchangeable. +package get_job_status + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + JobID string `json:"jobId" jsonschema:"UUID of the DGC job to check, as returned by start_ingestion. Do not pass an Edge job id here (e.g. from test_connection) — use edge_get_job_status for those instead."` +} + +type Output struct { + Name string `json:"name,omitempty" jsonschema:"The job's name."` + Type string `json:"type,omitempty" jsonschema:"The job's type (e.g. 'DELTA_INGESTION')."` + State string `json:"state,omitempty" jsonschema:"The job's current state: WAITING, RUNNING, CANCELING, COMPLETED, CANCELED, or ERROR."` + Result string `json:"result,omitempty" jsonschema:"The job's result once finished: NOT_SET (not finished yet), SUCCESS, COMPLETED_WITH_ERROR, FAILURE, or ABORTED."` + Message string `json:"message,omitempty" jsonschema:"Status message from the job."` + ProgressPercentage int `json:"progressPercentage,omitempty" jsonschema:"Progress percentage, 0-100."` + StartDate string `json:"startDate,omitempty"` + EndDate string `json:"endDate,omitempty" jsonschema:"When the job finished, if it has."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_job_status", + Title: "Get Job Status", + Description: "Gets the current status of a DGC job, e.g. the ingestion job started by start_ingestion. For Edge-site jobs (test_connection, capability runs), use edge_get_job_status instead — the two job id spaces are separate and not interchangeable.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("jobId", input.JobID); err != nil { + return Output{}, err + } + + job, err := clients.GetJob(ctx, collibraClient, input.JobID) + if err != nil { + return Output{}, err + } + + return Output{ + Name: job.Name, + Type: job.Type, + State: job.State, + Result: job.Result, + Message: job.Message, + ProgressPercentage: job.ProgressPercentage, + StartDate: job.StartDate, + EndDate: job.EndDate, + }, nil + } +} diff --git a/pkg/tools/get_job_status/tool_test.go b/pkg/tools/get_job_status/tool_test.go new file mode 100644 index 0000000..e0a586e --- /dev/null +++ b/pkg/tools/get_job_status/tool_test.go @@ -0,0 +1,41 @@ +package get_job_status_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/get_job_status" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestGetCatalogJobStatus(t *testing.T) { + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/jobs/v1/jobs/"+jobID.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"` + jobID.String() + `","name":"Database Synchronisation of source","type":"DELTA_INGESTION","state":"COMPLETED","result":"SUCCESS","progressPercentage":100}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: jobID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.State != "COMPLETED" || output.Result != "SUCCESS" { + t.Fatalf("unexpected output: %+v", output) + } +} + +func TestGetCatalogJobStatus_InvalidJobID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid jobId") + } +} diff --git a/pkg/tools/jobs_find/tool.go b/pkg/tools/jobs_find/tool.go new file mode 100644 index 0000000..8d0ee57 --- /dev/null +++ b/pkg/tools/jobs_find/tool.go @@ -0,0 +1,51 @@ +package jobs_find + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name,omitempty" jsonschema:"filter by job name"` + NameMatchMode string `json:"nameMatchMode,omitempty" jsonschema:"name match mode: EXACT or ANYWHERE"` + Result []string `json:"result,omitempty" jsonschema:"filter by result: NOT_SET, SUCCESS, COMPLETED_WITH_ERROR, FAILURE, ABORTED"` + State []string `json:"state,omitempty" jsonschema:"filter by state: WAITING, RUNNING, COMPLETED, FAILED, DELETED"` + Type []string `json:"type,omitempty" jsonschema:"filter by job type"` + User string `json:"user,omitempty" jsonschema:"filter by user who created the job"` + SortField string `json:"sortField,omitempty" jsonschema:"field to sort by"` + SortOrder string `json:"sortOrder,omitempty" jsonschema:"sort order: ASC or DESC"` + Cursor string `json:"cursor,omitempty" jsonschema:"pagination cursor from previous response"` + PageSize int `json:"pageSize,omitempty" jsonschema:"number of results per page"` +} + +type Output struct { + Jobs []clients.Job `json:"jobs,omitempty" jsonschema:"list of matching jobs"` + Count int `json:"count" jsonschema:"number of jobs returned"` + NextCursor string `json:"nextCursor,omitempty" jsonschema:"cursor for the next page of results"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "jobs_find", + Description: "Finds jobs matching the given filter criteria in the Collibra Jobs API.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + resp, err := clients.FindJobs(ctx, collibraClient, input.Name, input.NameMatchMode, input.Result, input.State, input.Type, input.User, input.SortField, input.SortOrder, input.Cursor, input.PageSize) + if err != nil { + return Output{Error: fmt.Sprintf("failed to find jobs: %s", err.Error())}, nil + } + return Output{Jobs: resp.Results, Count: len(resp.Results), NextCursor: resp.NextCursor}, nil + } +} diff --git a/pkg/tools/jobs_find/tool_test.go b/pkg/tools/jobs_find/tool_test.go new file mode 100644 index 0000000..aa50fbc --- /dev/null +++ b/pkg/tools/jobs_find/tool_test.go @@ -0,0 +1,74 @@ +package jobs_find_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/jobs_find" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestJobsFind(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobPagedResponse) { + return http.StatusOK, clients.JobPagedResponse{ + Results: []clients.Job{ + {ID: "job-1", Name: "test-job", State: "COMPLETED"}, + }, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Name: "test-job", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("expected no error, got: %s", output.Error) + } + if output.Count != 1 || output.Jobs[0].ID != "job-1" { + t.Fatal("expected one job with id job-1") + } +} + +func TestJobsFindAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "internal error" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected error message in output") + } +} + +func TestJobsFindNoFilters(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobPagedResponse) { + return http.StatusOK, clients.JobPagedResponse{Results: []clients.Job{}} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 0 { + t.Fatalf("expected 0 jobs, got %d", output.Count) + } +} diff --git a/pkg/tools/list_integrations/tool.go b/pkg/tools/list_integrations/tool.go new file mode 100644 index 0000000..7533e87 --- /dev/null +++ b/pkg/tools/list_integrations/tool.go @@ -0,0 +1,205 @@ +package list_integrations + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Platform string `json:"platform,omitempty" jsonschema:"optional platform filter: databricks, dataplex, etc. Matches type.Id substring. If omitted, all capabilities are returned."` + NameContains string `json:"nameContains,omitempty" jsonschema:"optional case-insensitive substring filter on capability name"` +} + +type Integration struct { + IngestibleId string `json:"ingestibleId" jsonschema:"UUID of the integration instance, use this for all subsequent calls"` + Name string `json:"name" jsonschema:"capability name"` + TypeId string `json:"typeId" jsonschema:"capability type, e.g. databricks-edge-capability"` + EdgeSiteId string `json:"edgeSiteId" jsonschema:"Edge site this capability runs on"` + HasSchedule bool `json:"hasSchedule" jsonschema:"whether a sync schedule is configured"` + CronExpression string `json:"cronExpression,omitempty" jsonschema:"cron expression for the schedule"` + CronTimeZone string `json:"cronTimeZone,omitempty" jsonschema:"timezone for the cron schedule"` + LastRun string `json:"lastRun,omitempty" jsonschema:"relative time of last sync run, e.g. 6h ago"` + LastRunAt string `json:"lastRunAt,omitempty" jsonschema:"ISO 8601 timestamp of last sync run, use this for precise time comparisons e.g. ran in last 24h"` + LastRunState string `json:"lastRunState,omitempty" jsonschema:"state of last job: COMPLETED, FAILED, RUNNING, etc."` + LastRunResult string `json:"lastRunResult,omitempty" jsonschema:"result of last job: SUCCESS, FAILURE, COMPLETED_WITH_ERROR, etc."` + NextRun string `json:"nextRun,omitempty" jsonschema:"relative time of next scheduled run, e.g. in 18h"` +} + +type Output struct { + Integrations []Integration `json:"integrations" jsonschema:"list of integrations with schedule info"` + Total int `json:"total" jsonschema:"total number of integrations returned"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "list_integrations", + Description: `Lists all integration capabilities with their sync schedule and last/next run times. +Use this as the entry point for any integration question: "show me all integrations", "show Databricks integrations", "what's scheduled for the Finance integration?". +Optionally filter by platform substring (e.g. "databricks", "dataplex") and/or name. Returns ingestibleId for each — pass it to catalog_etl_get_schedule, catalog_etl_start_job, or catalog_etl_cancel_job. +For time-based queries like "which integrations ran in the last 24h", use jobs_find sorted by startDate to get recent jobs, then cross-reference with integration names from this tool.`, + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + caps, err := clients.ListCapabilities(ctx, collibraClient) + if err != nil { + return Output{Error: fmt.Sprintf("failed to list capabilities: %s", err.Error())}, nil + } + + filtered := filterCapabilities(caps, input) + integrations := enrichWithSchedules(ctx, collibraClient, filtered) + return Output{Integrations: integrations, Total: len(integrations)}, nil + } +} + +func filterCapabilities(caps []clients.EdgeCapability, input Input) []clients.EdgeCapability { + platform := strings.ToLower(input.Platform) + name := strings.ToLower(input.NameContains) + + result := make([]clients.EdgeCapability, 0, len(caps)) + for _, c := range caps { + if platform != "" && platform != "all" { + typeId := "" + if c.Type != nil { + typeId = strings.ToLower(c.Type.Id) + } + if !strings.Contains(typeId, platform) { + continue + } + } + + if name != "" && !strings.Contains(strings.ToLower(c.Name), name) { + continue + } + + result = append(result, c) + } + return result +} + +func enrichWithSchedules(ctx context.Context, collibraClient *http.Client, caps []clients.EdgeCapability) []Integration { + integrations := make([]Integration, len(caps)) + now := time.Now() + + var wg sync.WaitGroup + for i, c := range caps { + wg.Add(1) + go func(idx int, cap clients.EdgeCapability) { + defer wg.Done() + integrations[idx] = buildIntegration(ctx, collibraClient, cap, now) + }(i, c) + } + wg.Wait() + return integrations +} + +func buildIntegration(ctx context.Context, collibraClient *http.Client, cap clients.EdgeCapability, now time.Time) Integration { + typeId := "" + if cap.Type != nil { + typeId = cap.Type.Id + } + + integration := Integration{ + IngestibleId: cap.Id, + Name: cap.Name, + TypeId: typeId, + EdgeSiteId: cap.EdgeSiteId, + } + + schedule, err := clients.GetGenericSchedule(ctx, collibraClient, cap.Id) + if err == nil && schedule != nil { + integration.HasSchedule = true + integration.CronExpression = schedule.CronExpression + integration.CronTimeZone = schedule.CronTimeZone + integration.NextRun = futureRelativeTime(schedule.NextRunDateLongValue, now) + if schedule.LastRunTimeStamp > 0 { + t := time.UnixMilli(schedule.LastRunTimeStamp) + integration.LastRun = relativeTime(t, now) + integration.LastRunAt = t.UTC().Format(time.RFC3339) + } + } + + // fall back to Jobs API for last run when schedule has no timestamp + if integration.LastRun == "" { + fillLastRunFromJobs(ctx, collibraClient, &integration, cap.Name, now) + } + + return integration +} + +func fillLastRunFromJobs(ctx context.Context, collibraClient *http.Client, integration *Integration, capName string, now time.Time) { + resp, err := clients.FindJobs(ctx, collibraClient, capName, "ANYWHERE", nil, nil, nil, "", "START_DATE", "DESC", "", 1) + if err != nil || len(resp.Results) == 0 { + return + } + job := resp.Results[0] + integration.LastRunState = job.State + integration.LastRunResult = job.Result + + // prefer EndDate for completed jobs, fall back to StartDate + dateStr := job.EndDate + if dateStr == "" { + dateStr = job.StartDate + } + if t := parseJobDate(dateStr); !t.IsZero() { + integration.LastRun = relativeTime(t, now) + integration.LastRunAt = t.UTC().Format(time.RFC3339) + } +} + +func parseJobDate(s string) time.Time { + if s == "" { + return time.Time{} + } + for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02T15:04:05.999Z07:00"} { + if t, err := time.Parse(layout, s); err == nil { + return t + } + } + return time.Time{} +} + +func relativeTime(t time.Time, now time.Time) string { + diff := now.Sub(t) + switch { + case diff < time.Minute: + return "just now" + case diff < time.Hour: + return fmt.Sprintf("%dm ago", int(diff.Minutes())) + case diff < 24*time.Hour: + return fmt.Sprintf("%dh ago", int(diff.Hours())) + default: + return fmt.Sprintf("%dd ago", int(diff.Hours()/24)) + } +} + +func futureRelativeTime(epochMs int64, now time.Time) string { + if epochMs == 0 { + return "" + } + diff := time.UnixMilli(epochMs).Sub(now) + if diff <= 0 { + return "overdue" + } + switch { + case diff < time.Hour: + return fmt.Sprintf("in %dm", int(diff.Minutes())) + case diff < 24*time.Hour: + return fmt.Sprintf("in %dh", int(diff.Hours())) + default: + return fmt.Sprintf("in %dd", int(diff.Hours()/24)) + } +} diff --git a/pkg/tools/list_integrations/tool_test.go b/pkg/tools/list_integrations/tool_test.go new file mode 100644 index 0000000..71895d6 --- /dev/null +++ b/pkg/tools/list_integrations/tool_test.go @@ -0,0 +1,217 @@ +package list_integrations_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/list_integrations" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestListIntegrations(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Sales", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "00000000-0000-0000-0000-000000000002", Name: "UC / Finance", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "00000000-0000-0000-0000-000000000003", Name: "Dataplex Finance", Type: &clients.EdgeCapabilityType{Id: "dataplex-synchronization"}}, + {Id: "00000000-0000-0000-0000-000000000004", Name: "jdbc-only", Type: &clients.EdgeCapabilityType{Id: "jdbc-profiling"}}, + } + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{CronExpression: "0 2 * * *", CronTimeZone: "UTC", LastRunTimeStamp: 1000, NextRunDateLongValue: 9999999999000} + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000002/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{CronExpression: "0 3 * * *", CronTimeZone: "UTC"} + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000003/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "no schedule" + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000004/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "no schedule" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + // all capabilities returned when no platform filter + if output.Total != 4 { + t.Fatalf("expected 4 integrations (all types), got %d", output.Total) + } +} + +func TestListIntegrationsFilterByPlatform(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Sales", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "00000000-0000-0000-0000-000000000002", Name: "Dataplex Finance", Type: &clients.EdgeCapabilityType{Id: "dataplex-synchronization"}}, + } + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{CronExpression: "0 2 * * *"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Platform: "databricks"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Total != 1 { + t.Fatalf("expected 1 databricks integration, got %d", output.Total) + } + if output.Integrations[0].Name != "UC / Sales" { + t.Fatalf("expected UC / Sales, got %s", output.Integrations[0].Name) + } +} + +func TestListIntegrationsScheduleEnrichment(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Sales", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + } + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{ + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + LastRunTimeStamp: 1000, + NextRunDateLongValue: 9999999999000, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Total != 1 { + t.Fatalf("expected 1 integration, got %d", output.Total) + } + integration := output.Integrations[0] + if !integration.HasSchedule { + t.Fatal("expected HasSchedule to be true") + } + if integration.CronExpression != "0 2 * * *" { + t.Fatalf("expected cron 0 2 * * *, got %s", integration.CronExpression) + } + if integration.LastRun == "" { + t.Fatal("expected LastRun to be set") + } + if integration.NextRun == "" { + t.Fatal("expected NextRun to be set") + } +} + +func TestListIntegrationsNoScheduleFallsBackToJobs(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Marketing", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + } + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobPagedResponse) { + return http.StatusOK, clients.JobPagedResponse{ + Results: []clients.Job{ + {ID: "job-1", Name: "UC / Marketing", State: "COMPLETED", Result: "SUCCESS", EndDate: "2026-05-01T02:00:00Z"}, + }, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Total != 1 { + t.Fatalf("expected 1 integration, got %d", output.Total) + } + integration := output.Integrations[0] + if integration.HasSchedule { + t.Fatal("expected HasSchedule to be false") + } + if integration.LastRun == "" { + t.Fatal("expected LastRun to be populated from jobs API") + } + if integration.LastRunState != "COMPLETED" { + t.Fatalf("expected LastRunState COMPLETED, got %s", integration.LastRunState) + } + if integration.LastRunResult != "SUCCESS" { + t.Fatalf("expected LastRunResult SUCCESS, got %s", integration.LastRunResult) + } +} + +func TestListIntegrationsScheduleNoLastRunFallsBackToJobs(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Sales", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + } + })) + // schedule exists but lastRunTimeStamp is 0 (never run) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{CronExpression: "0 2 * * *", CronTimeZone: "UTC", LastRunTimeStamp: 0} + })) + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobPagedResponse) { + return http.StatusOK, clients.JobPagedResponse{ + Results: []clients.Job{ + {ID: "job-1", State: "FAILED", Result: "FAILURE", EndDate: "2026-04-30T03:00:00Z"}, + }, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + integration := output.Integrations[0] + if !integration.HasSchedule { + t.Fatal("expected HasSchedule to be true") + } + if integration.LastRun == "" { + t.Fatal("expected LastRun from jobs fallback") + } + if integration.LastRunState != "FAILED" { + t.Fatalf("expected LastRunState FAILED, got %s", integration.LastRunState) + } +} + +func TestListIntegrationsAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "error" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error to be set") + } +} diff --git a/pkg/tools/register.go b/pkg/tools/register.go index fee28b6..edf6d62 100644 --- a/pkg/tools/register.go +++ b/pkg/tools/register.go @@ -7,15 +7,49 @@ import ( "github.com/collibra/chip/pkg/chip" "github.com/collibra/chip/pkg/skills" "github.com/collibra/chip/pkg/tools/add_data_classification_match" + "github.com/collibra/chip/pkg/tools/catalog_etl_add_schedule" + "github.com/collibra/chip/pkg/tools/catalog_etl_cancel_job" + "github.com/collibra/chip/pkg/tools/catalog_etl_delete_generic_config" + "github.com/collibra/chip/pkg/tools/catalog_etl_delete_schedule" + "github.com/collibra/chip/pkg/tools/catalog_etl_get_all_schedules" + "github.com/collibra/chip/pkg/tools/catalog_etl_get_config" + "github.com/collibra/chip/pkg/tools/catalog_etl_get_schedule" + "github.com/collibra/chip/pkg/tools/catalog_etl_get_schema" + "github.com/collibra/chip/pkg/tools/catalog_etl_save_generic_config" + "github.com/collibra/chip/pkg/tools/catalog_etl_start_job" + "github.com/collibra/chip/pkg/tools/catalog_etl_update_schedule" + "github.com/collibra/chip/pkg/tools/configure_database_schemas" "github.com/collibra/chip/pkg/tools/create_asset" + "github.com/collibra/chip/pkg/tools/create_community" + "github.com/collibra/chip/pkg/tools/create_domain" "github.com/collibra/chip/pkg/tools/discover_business_glossary" "github.com/collibra/chip/pkg/tools/discover_data_assets" + "github.com/collibra/chip/pkg/tools/edge_cancel_job" + "github.com/collibra/chip/pkg/tools/edge_create_capability" + "github.com/collibra/chip/pkg/tools/edge_create_connection" + "github.com/collibra/chip/pkg/tools/edge_delete_capability" + "github.com/collibra/chip/pkg/tools/edge_delete_connection" + "github.com/collibra/chip/pkg/tools/edge_find_capabilities" + "github.com/collibra/chip/pkg/tools/edge_find_connections" + "github.com/collibra/chip/pkg/tools/edge_get_capability" + "github.com/collibra/chip/pkg/tools/edge_get_connection" + "github.com/collibra/chip/pkg/tools/edge_get_job_status" + "github.com/collibra/chip/pkg/tools/edge_get_job_status_history" + "github.com/collibra/chip/pkg/tools/edge_list_capabilities" + "github.com/collibra/chip/pkg/tools/edge_list_capability_types" + "github.com/collibra/chip/pkg/tools/edge_list_connections" + "github.com/collibra/chip/pkg/tools/edge_list_sites" + "github.com/collibra/chip/pkg/tools/edge_run_capability" "github.com/collibra/chip/pkg/tools/edit_asset" + "github.com/collibra/chip/pkg/tools/find_domain_types" + "github.com/collibra/chip/pkg/tools/find_users" "github.com/collibra/chip/pkg/tools/get_asset_details" "github.com/collibra/chip/pkg/tools/get_business_term_data" "github.com/collibra/chip/pkg/tools/get_column_semantics" "github.com/collibra/chip/pkg/tools/get_context_specification" + "github.com/collibra/chip/pkg/tools/get_data_source_setup_guide" "github.com/collibra/chip/pkg/tools/get_debug_mcp_init_request" + "github.com/collibra/chip/pkg/tools/get_job_status" "github.com/collibra/chip/pkg/tools/get_lineage_downstream" "github.com/collibra/chip/pkg/tools/get_lineage_entity" "github.com/collibra/chip/pkg/tools/get_lineage_transformation" @@ -23,18 +57,24 @@ import ( "github.com/collibra/chip/pkg/tools/get_measure_data" "github.com/collibra/chip/pkg/tools/get_table_semantics" "github.com/collibra/chip/pkg/tools/init_data_contract" + "github.com/collibra/chip/pkg/tools/jobs_find" "github.com/collibra/chip/pkg/tools/list_asset_types" "github.com/collibra/chip/pkg/tools/list_context_specifications" "github.com/collibra/chip/pkg/tools/list_data_contracts" + "github.com/collibra/chip/pkg/tools/list_integrations" "github.com/collibra/chip/pkg/tools/prepare_create_asset" "github.com/collibra/chip/pkg/tools/pull_data_contract_manifest" "github.com/collibra/chip/pkg/tools/push_data_contract_manifest" + "github.com/collibra/chip/pkg/tools/register_database" "github.com/collibra/chip/pkg/tools/remove_data_classification_match" "github.com/collibra/chip/pkg/tools/search_asset_keyword" "github.com/collibra/chip/pkg/tools/search_data_classes" "github.com/collibra/chip/pkg/tools/search_data_classification_matches" "github.com/collibra/chip/pkg/tools/search_lineage_entities" "github.com/collibra/chip/pkg/tools/search_lineage_transformations" + "github.com/collibra/chip/pkg/tools/start_ingestion" + "github.com/collibra/chip/pkg/tools/test_connection" + "github.com/collibra/chip/pkg/tools/upload_file" ) // ContextSpecificationsFeature is the experimental-feature identifier used to @@ -80,6 +120,57 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, list_context_specifications.NewTool(client)) toolRegister(server, toolConfig, get_context_specification.NewTool(client)) } + toolRegister(server, toolConfig, find_users.NewTool(client)) + toolRegister(server, toolConfig, find_domain_types.NewTool(client)) + toolRegister(server, toolConfig, edge_find_connections.NewTool(client)) + toolRegister(server, toolConfig, edge_list_sites.NewTool(client)) + toolRegister(server, toolConfig, edge_list_capability_types.NewTool(client)) + toolRegister(server, toolConfig, create_community.NewTool(client)) + toolRegister(server, toolConfig, create_domain.NewTool(client)) + if toolConfig.AllowLocalFileUpload { + toolRegister(server, toolConfig, upload_file.NewToolWithFilePath(client)) + } else { + toolRegister(server, toolConfig, upload_file.NewTool(client)) + } + toolRegister(server, toolConfig, get_data_source_setup_guide.NewTool(client)) + toolRegister(server, toolConfig, edge_create_connection.NewTool(client)) + toolRegister(server, toolConfig, edge_create_capability.NewTool(client)) + toolRegister(server, toolConfig, test_connection.NewTool(client)) + toolRegister(server, toolConfig, edge_get_job_status.NewTool(client)) + toolRegister(server, toolConfig, get_job_status.NewTool(client)) + toolRegister(server, toolConfig, register_database.NewTool(client)) + toolRegister(server, toolConfig, configure_database_schemas.NewTool(client)) + toolRegister(server, toolConfig, start_ingestion.NewTool(client)) + + // Edge Management tools — read/delete/run operations with no counterpart among the + // create/find/test/get-status lifecycle tools above (edge_create_connection and + // edge_create_capability already upsert, so no edge update tools are needed here). + toolRegister(server, toolConfig, edge_list_capabilities.NewTool(client)) + toolRegister(server, toolConfig, edge_find_capabilities.NewTool(client)) + toolRegister(server, toolConfig, edge_get_capability.NewTool(client)) + toolRegister(server, toolConfig, edge_delete_capability.NewTool(client)) + toolRegister(server, toolConfig, edge_run_capability.NewTool(client)) + toolRegister(server, toolConfig, edge_list_connections.NewTool(client)) + toolRegister(server, toolConfig, edge_get_connection.NewTool(client)) + toolRegister(server, toolConfig, edge_delete_connection.NewTool(client)) + toolRegister(server, toolConfig, edge_cancel_job.NewTool(client)) + toolRegister(server, toolConfig, edge_get_job_status_history.NewTool(client)) + // Catalog Generic Integration tools + toolRegister(server, toolConfig, catalog_etl_get_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_save_generic_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_delete_generic_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_get_schema.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_get_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_add_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_update_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_delete_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_get_all_schedules.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_cancel_job.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_start_job.NewTool(client)) + // Jobs tools + toolRegister(server, toolConfig, jobs_find.NewTool(client)) + // Integration lifecycle tools + toolRegister(server, toolConfig, list_integrations.NewTool(client)) if toolConfig.EnableDebugTools { toolRegister(server, toolConfig, get_debug_mcp_init_request.NewTool(client)) diff --git a/pkg/tools/register_database/tool.go b/pkg/tools/register_database/tool.go new file mode 100644 index 0000000..a276291 --- /dev/null +++ b/pkg/tools/register_database/tool.go @@ -0,0 +1,145 @@ +// Package register_database implements the register_database MCP tool: discovers a +// database through an Edge connection and registers it as a DGC Database asset, via +// the public catalogDatabase API. +// +// This is the first of two steps in setting up a database for jdbc-ingestion — the +// second, configuring which schemas/tables get synchronized, is a separate tool +// (configure_database_schemas), which takes this tool's returned databaseConnectionId. They +// were originally one tool; split apart because registering the database is a +// one-time, id-claiming operation (POST /databases permanently claims the underlying +// database connection — a second call for the same one fails with +// databaseConnectionAlreadyUsed) while schema configuration is something an agent may +// reasonably want to redo independently, and keeping them separate means a +// schema-selection ambiguity error here never risks colliding with a database this +// tool already registered. +package register_database + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// Sleep is used for polling backoff. Overridable in tests to avoid slow test runs. +var Sleep = time.Sleep + +const ( + // pollAttempts * pollInterval bounds the wait for the async refresh. A + // first-time connection to a data source can take 30-60s to establish, so + // this budget needs enough headroom to cover that cold-start case. + pollAttempts = 14 + pollInterval = 5 * time.Second +) + +type Input struct { + EdgeConnectionID string `json:"edgeConnectionId" jsonschema:"UUID of the Edge connection (created via edge_create_connection) to discover a database on."` + DatabaseName string `json:"databaseName,omitempty" jsonschema:"Optional. Exact name of the database (catalog) to register, as it appears at the data source. Required if the data source exposes more than one database/catalog through this connection; if there is exactly one, it is selected automatically."` + CommunityID string `json:"communityId" jsonschema:"UUID of the community the Database asset (and its automatically created domain) will be created in."` + ParentSystemID string `json:"parentSystemId" jsonschema:"UUID of the parent System asset the Database asset will be linked to."` + OwnerIDs []string `json:"ownerIds" jsonschema:"UUIDs of the users to assign as owners of the Database asset. Use find_users to resolve a name (e.g. 'Admin') to its UUID."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the Database asset."` +} + +type Output struct { + Database *clients.Database `json:"database,omitempty" jsonschema:"The registered Database asset. Pass its databaseConnectionId to configure_database_schemas next."` + Success bool `json:"success" jsonschema:"Whether the database was registered successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if registration failed or is still in progress (safe to retry)."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "register_database", + Title: "Register Database for Ingestion", + Description: "Discovers a database through an Edge connection and registers it as a Database asset. Prerequisite for configure_database_schemas (which sets up schema/table synchronization) and, transitively, start_ingestion. Assumes the target community and parent System asset already exist (create_community/create_domain/create_asset), and that a jdbc-ingestion capability referencing this connection has already been created via edge_create_capability — discovery only finds data because that capability actually performs the crawl; without one, this fails with a discovery-timeout-shaped error even though the real cause is the missing capability. If more than one database is discovered, this returns an error naming the candidates instead of guessing — confirm which one with the user.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("edgeConnectionId", input.EdgeConnectionID); err != nil { + return Output{}, err + } + if err := validation.UUID("communityId", input.CommunityID); err != nil { + return Output{}, err + } + if err := validation.UUID("parentSystemId", input.ParentSystemID); err != nil { + return Output{}, err + } + if len(input.OwnerIDs) == 0 { + return Output{}, fmt.Errorf("ownerIds must contain at least one user UUID — use find_users to resolve a name to an id") + } + if err := validation.UUIDs("ownerIds", input.OwnerIDs); err != nil { + return Output{}, err + } + + databaseConnection, err := discoverDatabaseConnection(ctx, collibraClient, input.EdgeConnectionID, input.DatabaseName) + if err != nil { + return Output{Success: false, Error: err.Error()}, nil + } + + database, err := clients.RegisterDatabase(ctx, collibraClient, clients.AddDatabaseRequest{ + DatabaseConnectionID: databaseConnection.ID, + CommunityID: input.CommunityID, + ParentSystemID: input.ParentSystemID, + OwnerIDs: input.OwnerIDs, + Description: input.Description, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to register database: %s", err.Error())}, nil + } + + return Output{Database: database, Success: true}, nil + } +} + +// discoverDatabaseConnection triggers a refresh of database connections for the given +// Edge connection and polls until at least one appears, then selects one by name (or +// the only one, if there's no ambiguity). +func discoverDatabaseConnection(ctx context.Context, client *http.Client, edgeConnectionID, databaseName string) (*clients.DatabaseConnection, error) { + if err := clients.RefreshDatabaseConnections(ctx, client, edgeConnectionID); err != nil { + return nil, fmt.Errorf("failed to refresh database connections: %w", err) + } + + var connections []clients.DatabaseConnection + for attempt := 0; attempt < pollAttempts; attempt++ { + if attempt > 0 { + Sleep(pollInterval) + } + found, err := clients.ListDatabaseConnections(ctx, client, edgeConnectionID) + if err != nil { + return nil, fmt.Errorf("failed to list database connections: %w", err) + } + if len(found) > 0 { + connections = found + break + } + } + + if len(connections) == 0 { + return nil, fmt.Errorf("no database connections were discovered for edge connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or — more commonly — no capability referencing this connection exists yet: the discovery/refresh only finds data because a jdbc-ingestion capability actually performs the crawl. Verify a capability exists for this connection (edge_create_capability) before retrying", edgeConnectionID, pollAttempts) + } + + if databaseName != "" { + for _, connection := range connections { + if connection.Name == databaseName { + return &connection, nil + } + } + return nil, fmt.Errorf("no database named %q found among discovered database connections for edge connection %s", databaseName, edgeConnectionID) + } + + if len(connections) > 1 { + return nil, fmt.Errorf("multiple databases discovered for edge connection %s; specify databaseName to select one", edgeConnectionID) + } + + return &connections[0], nil +} diff --git a/pkg/tools/register_database/tool_test.go b/pkg/tools/register_database/tool_test.go new file mode 100644 index 0000000..3386fb8 --- /dev/null +++ b/pkg/tools/register_database/tool_test.go @@ -0,0 +1,188 @@ +package register_database_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/register_database" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func init() { + tools.Sleep = func(time.Duration) {} +} + +func TestRegisterDatabase_Success(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + dbConnID, _ := uuid.NewUUID() + databaseID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("edgeConnectionId") != edgeConnID.String() { + t.Fatalf("unexpected edgeConnectionId: %s", r.URL.Query().Get("edgeConnectionId")) + } + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.DatabaseConnection{ + {ID: dbConnID.String(), Name: "source", EdgeConnectionID: edgeConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { + if in.DatabaseConnectionID != dbConnID.String() { + t.Fatalf("unexpected databaseConnectionId: %s", in.DatabaseConnectionID) + } + return http.StatusCreated, clients.Database{ + ID: databaseID.String(), + Name: "source", + CommunityID: in.CommunityID, + OwnerIDs: in.OwnerIDs, + ParentSystemID: in.ParentSystemID, + DatabaseConnectionID: in.DatabaseConnectionID, + } + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Database.ID != databaseID.String() { + t.Fatalf("expected database id %s, got %s", databaseID.String(), output.Database.ID) + } + if output.Database.DatabaseConnectionID != dbConnID.String() { + t.Fatalf("expected databaseConnectionId %s, got %s", dbConnID.String(), output.Database.DatabaseConnectionID) + } +} + +func TestRegisterDatabase_MultipleDatabasesRequireDatabaseName(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + dbConnID1, _ := uuid.NewUUID() + dbConnID2, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.DatabaseConnection{ + {ID: dbConnID1.String(), Name: "source1", EdgeConnectionID: edgeConnID.String()}, + {ID: dbConnID2.String(), Name: "source2", EdgeConnectionID: edgeConnID.String()}, + }, + } + })) + databaseRegistered := false + mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { + databaseRegistered = true + return http.StatusCreated, clients.Database{DatabaseConnectionID: in.DatabaseConnectionID} + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err != nil { + t.Fatalf("expected no error (failures reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when multiple databases are discovered without databaseName") + } + if output.Error == "" { + t.Fatalf("expected an error message naming the discovered databases") + } + if databaseRegistered { + t.Fatalf("expected no Database asset to be registered when database selection is ambiguous") + } +} + +func TestRegisterDatabase_NoDatabaseConnectionsDiscovered(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{"results": []clients.DatabaseConnection{}} + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err != nil { + t.Fatalf("expected no error (failures reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when no database connections are discovered") + } + if output.Error == "" { + t.Fatalf("expected an error message") + } +} + +func TestRegisterDatabase_InvalidInput(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: "not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid edgeConnectionId") + } +} + +func TestRegisterDatabase_RequiresOwnerIDs(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + }) + if err == nil { + t.Fatalf("expected an error when ownerIds is empty") + } +} diff --git a/pkg/tools/start_ingestion/tool.go b/pkg/tools/start_ingestion/tool.go new file mode 100644 index 0000000..5b02410 --- /dev/null +++ b/pkg/tools/start_ingestion/tool.go @@ -0,0 +1,54 @@ +// Package start_ingestion implements the start_ingestion MCP tool: triggers the +// jdbc-ingestion capability run for a registered Database asset. +package start_ingestion + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + DatabaseID string `json:"databaseId" jsonschema:"UUID of the Database asset to synchronize (as registered by register_database)."` + SchemaConnectionIDs []string `json:"schemaConnectionIds,omitempty" jsonschema:"Optional. UUIDs of specific schema connections to synchronize. If omitted, all schemas that already have synchronization rules configured (via configure_database_schemas) are synchronized."` +} + +type Output struct { + Job *clients.CatalogJob `json:"job,omitempty" jsonschema:"The started synchronization job. Poll its id with get_job_status (NOT edge_get_job_status, which is for Edge-site jobs) to see when it completes."` + Success bool `json:"success" jsonschema:"Whether the ingestion job was started."` + Error string `json:"error,omitempty" jsonschema:"Error message if starting the job failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "start_ingestion", + Title: "Start Database Ingestion", + Description: "Triggers the jdbc-ingestion capability run for a registered Database asset, synchronizing its configured schemas/tables into the catalog. The database must already be registered (register_database) and have its schemas configured (configure_database_schemas). Poll the returned job's id with get_job_status to confirm it actually completed — a 202/success response here only means the job was accepted, not that ingestion finished.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("databaseId", input.DatabaseID); err != nil { + return Output{}, err + } + if err := validation.UUIDs("schemaConnectionIds", input.SchemaConnectionIDs); err != nil { + return Output{}, err + } + + job, err := clients.SynchronizeDatabaseMetadata(ctx, collibraClient, input.DatabaseID, input.SchemaConnectionIDs) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to start ingestion: %s", err.Error())}, nil + } + + return Output{Job: job, Success: true}, nil + } +} diff --git a/pkg/tools/start_ingestion/tool_test.go b/pkg/tools/start_ingestion/tool_test.go new file mode 100644 index 0000000..1061576 --- /dev/null +++ b/pkg/tools/start_ingestion/tool_test.go @@ -0,0 +1,103 @@ +package start_ingestion_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/start_ingestion" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestStartIngestion_Success(t *testing.T) { + databaseID, _ := uuid.NewUUID() + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /rest/catalogDatabase/v1/databases/"+databaseID.String()+"/synchronizeMetadata", + testutil.JsonHandlerInOut(func(r *http.Request, in clients.DatabaseMetadataSynchronizationRequest) (int, clients.CatalogJob) { + if len(in.SchemaConnectionIDs) != 0 { + t.Fatalf("expected no schemaConnectionIds, got: %v", in.SchemaConnectionIDs) + } + return http.StatusAccepted, clients.CatalogJob{ID: jobID.String(), State: "RUNNING"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseID: databaseID.String(), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Job.ID != jobID.String() { + t.Fatalf("expected job id %s, got %s", jobID.String(), output.Job.ID) + } +} + +func TestStartIngestion_WithSchemaConnectionIDs(t *testing.T) { + databaseID, _ := uuid.NewUUID() + schemaConnID, _ := uuid.NewUUID() + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /rest/catalogDatabase/v1/databases/"+databaseID.String()+"/synchronizeMetadata", + testutil.JsonHandlerInOut(func(r *http.Request, in clients.DatabaseMetadataSynchronizationRequest) (int, clients.CatalogJob) { + if len(in.SchemaConnectionIDs) != 1 || in.SchemaConnectionIDs[0] != schemaConnID.String() { + t.Fatalf("unexpected schemaConnectionIds: %v", in.SchemaConnectionIDs) + } + return http.StatusAccepted, clients.CatalogJob{ID: jobID.String(), State: "RUNNING"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseID: databaseID.String(), + SchemaConnectionIDs: []string{schemaConnID.String()}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestStartIngestion_InvalidDatabaseID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{DatabaseID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid databaseId") + } +} + +func TestStartIngestion_Conflict(t *testing.T) { + databaseID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("POST /rest/catalogDatabase/v1/databases/"+databaseID.String()+"/synchronizeMetadata", + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusConflict) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{DatabaseID: databaseID.String()}) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure on conflict") + } +} diff --git a/pkg/tools/test_connection/tool.go b/pkg/tools/test_connection/tool.go new file mode 100644 index 0000000..94a9873 --- /dev/null +++ b/pkg/tools/test_connection/tool.go @@ -0,0 +1,57 @@ +// Package test_connection implements the test_connection MCP tool: verifies an Edge +// connection can actually reach its data source. +package test_connection + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + ConnectionID string `json:"connectionId" jsonschema:"UUID of the connection to test (created via edge_create_connection)."` + TimeoutSec int `json:"timeoutSec,omitempty" jsonschema:"Optional. If provided and greater than 0, waits synchronously for the test to finish (up to this many seconds) and returns the final result. If omitted, returns immediately with a jobId while the test runs in the background — poll it with edge_get_job_status."` +} + +type Output struct { + JobID string `json:"jobId,omitempty" jsonschema:"The id of the connection-test job. Use with edge_get_job_status to poll for the result if timeoutSec was not provided."` + Success bool `json:"success" jsonschema:"Whether the test connection request itself succeeded. If timeoutSec was omitted, this reflects that the test job was submitted, not the connection test's outcome — check edge_get_job_status for that."` + Message string `json:"message,omitempty" jsonschema:"Status message from the test."` + Error string `json:"error,omitempty" jsonschema:"Error message if the test request itself failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "test_connection", + Title: "Test Edge Connection", + Description: "Tests whether an Edge connection can reach its data source. Without timeoutSec, returns immediately with a jobId to poll via edge_get_job_status; with timeoutSec, waits up to that many seconds and returns the final result.", + Handler: handler(collibraClient), + Permissions: []string{"dgc.edge-integration-capability-manage"}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("connectionId", input.ConnectionID); err != nil { + return Output{}, err + } + + var timeoutSec *int + if input.TimeoutSec > 0 { + timeoutSec = &input.TimeoutSec + } + + result, err := clients.TestConnection(ctx, collibraClient, input.ConnectionID, timeoutSec) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to test connection: %s", err.Error())}, nil + } + + return Output{JobID: result.JobID, Success: result.Success, Message: result.Message}, nil + } +} diff --git a/pkg/tools/test_connection/tool_test.go b/pkg/tools/test_connection/tool_test.go new file mode 100644 index 0000000..a25f386 --- /dev/null +++ b/pkg/tools/test_connection/tool_test.go @@ -0,0 +1,73 @@ +package test_connection_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/test_connection" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestTestConnection_Async(t *testing.T) { + connID, _ := uuid.NewUUID() + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/connections/"+connID.String()+"/test", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.TestConnectionResponse) { + if r.URL.Query().Get("timeoutSec") != "" { + t.Fatalf("expected no timeoutSec query param, got: %s", r.URL.Query().Get("timeoutSec")) + } + return http.StatusOK, clients.TestConnectionResponse{JobID: jobID.String(), Success: true, Message: "Job submitted successfully."} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ConnectionID: connID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success || output.JobID != jobID.String() { + t.Fatalf("unexpected output: %+v", output) + } +} + +func TestTestConnection_Synchronous(t *testing.T) { + connID, _ := uuid.NewUUID() + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/connections/"+connID.String()+"/test", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.TestConnectionResponse) { + if r.URL.Query().Get("timeoutSec") != "30" { + t.Fatalf("expected timeoutSec=30, got: %s", r.URL.Query().Get("timeoutSec")) + } + return http.StatusOK, clients.TestConnectionResponse{JobID: jobID.String(), Success: false, Message: "connection refused"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ConnectionID: connID.String(), TimeoutSec: 30}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Success { + t.Fatalf("expected the connection test itself to report failure") + } + if output.Message != "connection refused" { + t.Fatalf("unexpected message: %s", output.Message) + } +} + +func TestTestConnection_InvalidConnectionID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ConnectionID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid connectionId") + } +} diff --git a/pkg/tools/upload_file/tool.go b/pkg/tools/upload_file/tool.go new file mode 100644 index 0000000..7387707 --- /dev/null +++ b/pkg/tools/upload_file/tool.go @@ -0,0 +1,155 @@ +// Package upload_file implements the upload_file MCP tool: uploads a file to an +// edge site and returns an artifact URI usable as the value of any FILE-type +// connection or capability parameter (JDBC drivers, TLS certs, private keys, +// keytabs, etc.). +// +// Policy: this tool deliberately does NOT fetch file content from an arbitrary URL +// (e.g. Maven Central). For JDBC drivers specifically, Collibra only supports/ +// verifies drivers distributed through Collibra Marketplace, and Marketplace +// downloads require an authenticated account — there is no public, unauthenticated +// download endpoint to automate. get_data_source_setup_guide's marketplaceUrl field +// points to the correct listing; the user must download the driver from there +// themselves and hand the bytes to this tool. Do not substitute an unverified source +// (Maven Central, GitHub releases, etc.) for a driver jar just because it's more +// convenient to automate — ask the user for the file instead. +// +// Large files and the two Input variants (documented gap for chip-service): +// +// contentBase64 has a hard ceiling that isn't really about encoding or transport: any +// value passed as a tool-call argument has to be generated by the calling model +// itself, token by token, as part of its own output. A 96MB JDBC driver jar becomes +// ~130MB of base64 text the model would have to produce — that holds regardless of +// encoding, MCP transport, or how this tool is implemented. +// +// filePath (reading the file directly off local disk, server-side, bypassing the +// model entirely) fixes this — but only makes sense when chip is running as a local +// process with access to the caller's own filesystem. That's true for the standalone +// `chip` binary (a real, documented distribution channel — see the repo README — +// where a user runs chip locally against their own tenant), but not for chip-service, +// the product's remote, multi-tenant deployment: there, "local disk" means the server +// container's disk, not any user's machine, and a filePath input would be a genuine +// arbitrary-file-read vulnerability if ever reachable. +// +// Both binaries call the same tools.RegisterAll, so filePath support is exposed as a +// second, separate Input type (InputWithFilePath) registered under an unchanged tool +// name ("upload_file") only when chip.ServerToolConfig.AllowLocalFileUpload is true. +// This is gated at registration time, not inside the handler on every call: only +// cmd/chip's own startup wiring ever sets AllowLocalFileUpload, so chip-service's +// binary has no code path that could turn it on, even by mistake. If chip-service +// ever needs equivalent large-file support, it needs its own solution appropriate to +// a remote, multi-tenant server (e.g. a presigned upload step, or client-side file +// attachment support in whatever chat surface embeds it) — not this field. +// +// For chip-service today, the only large-file workaround is manual: guide the user +// to create the whole connection through the Collibra Edge UI (Settings > Edge > +// site > Connections > New), supplying the exact field values yourself (driver +// class, connection string — from get_data_source_setup_guide) so they only have to +// pick the file and click through, since there is no standalone "upload just the +// driver, get a reference back" screen or API — confirmed by checking the newer +// catalog-backed driver API (POST /rest/edge/v1/drivers/jars), which has no UI in +// front of it at all, and the frontend's only file-upload path is the legacy one +// embedded inside the connection-creation form's own FILE-type field. Once the user +// confirms the connection is saved, use edge_find_connections (by name) to pick up its +// id and proceed with edge_create_capability/register_database/ +// configure_database_schemas/start_ingestion against it — skip edge_create_connection +// entirely for that connection, since it already exists. +package upload_file + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "os" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// Input is used when local file uploads are not enabled (the default, and always the +// case for chip-service) — content must be supplied inline as base64. +type Input struct { + Filename string `json:"filename" jsonschema:"The filename to upload as (e.g. 'client-cert.pem', 'snowflake-jdbc-3.23.2.jar'). Determines the file extension DGC uses to infer the artifact type."` + ContentBase64 string `json:"contentBase64" jsonschema:"The file's content, base64-encoded. For a JDBC driver, this must come from a file the user downloaded from Collibra Marketplace (see get_data_source_setup_guide's marketplaceUrl) — never fetch a driver from an arbitrary URL like Maven Central; ask the user for the file instead. Large files (tens of MB, as most JDBC drivers are) will not fit here — if the content would be too large to include, ask the user to create the connection manually via the Collibra Edge UI instead and pick it up afterward with edge_find_connections."` +} + +// InputWithFilePath is used when AllowLocalFileUpload is true (only ever set by the +// standalone chip binary, never chip-service) — adds filePath as an alternative to +// contentBase64 for large files. +type InputWithFilePath struct { + Filename string `json:"filename" jsonschema:"The filename to upload as (e.g. 'client-cert.pem', 'snowflake-jdbc-3.23.2.jar'). Determines the file extension DGC uses to infer the artifact type."` + ContentBase64 string `json:"contentBase64,omitempty" jsonschema:"The file's content, base64-encoded. Exactly one of contentBase64 or filePath must be provided. Prefer filePath for large files (tens of MB, as most JDBC drivers are) — contentBase64 will not fit them in a tool call."` + FilePath string `json:"filePath,omitempty" jsonschema:"Absolute path to a local file to read and upload directly from this machine's disk, without passing its content through the conversation. Available because this chip instance is running locally with access to your own filesystem. Exactly one of contentBase64 or filePath must be provided."` +} + +type Output struct { + URI string `json:"uri,omitempty" jsonschema:"The uploaded artifact URI (e.g. 'jar:///'). Use this as the value of a FILE-type connection or capability parameter."` + Success bool `json:"success" jsonschema:"Whether the file was uploaded successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if the upload failed."` +} + +// NewTool registers upload_file without filePath support (contentBase64 only). Used +// whenever AllowLocalFileUpload is false — always true for chip-service. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "upload_file", + Title: "Upload File to Edge Site", + Description: "Uploads a file (JDBC driver, TLS certificate, private key, keytab, etc.) to an edge site and returns an artifact URI usable as the value of any FILE-type connection or capability parameter. Takes file content directly (base64) — does not fetch from a URL. Only suitable for small files (contentBase64 has to fit in a tool call); for a large file (e.g. most JDBC drivers), guide the user to create the connection manually via the Collibra Edge UI and pick it up with edge_find_connections. For JDBC drivers, only use content downloaded from Collibra Marketplace (verified/supported), never an arbitrary host; get the Marketplace listing URL from get_data_source_setup_guide.", + Handler: func(ctx context.Context, input Input) (Output, error) { + return upload(ctx, collibraClient, input.Filename, input.ContentBase64, "") + }, + Permissions: []string{"dgc.edge-integration-capability-manage"}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +// NewToolWithFilePath registers upload_file with filePath support added. Only called +// when chip.ServerToolConfig.AllowLocalFileUpload is true — see package doc comment +// for why this must never happen in chip-service. +func NewToolWithFilePath(collibraClient *http.Client) *chip.Tool[InputWithFilePath, Output] { + return &chip.Tool[InputWithFilePath, Output]{ + Name: "upload_file", + Title: "Upload File to Edge Site", + Description: "Uploads a file (JDBC driver, TLS certificate, private key, keytab, etc.) to an edge site and returns an artifact URI usable as the value of any FILE-type connection or capability parameter. Does not fetch from a URL. Prefer filePath for large files (e.g. most JDBC drivers) — it reads the file directly from this machine's disk, avoiding the base64-in-tool-call size limit. For JDBC drivers, only use a file downloaded from Collibra Marketplace (verified/supported), never an arbitrary host; get the Marketplace listing URL from get_data_source_setup_guide.", + Handler: func(ctx context.Context, input InputWithFilePath) (Output, error) { + return upload(ctx, collibraClient, input.Filename, input.ContentBase64, input.FilePath) + }, + Permissions: []string{"dgc.edge-integration-capability-manage"}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func upload(ctx context.Context, collibraClient *http.Client, filename, contentBase64, filePath string) (Output, error) { + if filename == "" { + return Output{}, fmt.Errorf("filename is required") + } + + hasContent := contentBase64 != "" + hasFilePath := filePath != "" + if hasContent == hasFilePath { + return Output{}, fmt.Errorf("exactly one of contentBase64 or filePath must be provided") + } + + var content []byte + if hasFilePath { + data, err := os.ReadFile(filePath) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to read file %q: %s", filePath, err.Error())}, nil + } + content = data + } else { + decoded, err := base64.StdEncoding.DecodeString(contentBase64) + if err != nil { + return Output{}, fmt.Errorf("decoding contentBase64: %w", err) + } + content = decoded + } + + uri, err := clients.UploadFile(ctx, collibraClient, filename, content) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to upload file: %s", err.Error())}, nil + } + + return Output{URI: uri, Success: true}, nil +} diff --git a/pkg/tools/upload_file/tool_test.go b/pkg/tools/upload_file/tool_test.go new file mode 100644 index 0000000..fe4e8a9 --- /dev/null +++ b/pkg/tools/upload_file/tool_test.go @@ -0,0 +1,127 @@ +package upload_file_test + +import ( + "encoding/base64" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/collibra/chip/pkg/tools/testutil" + tools "github.com/collibra/chip/pkg/tools/upload_file" +) + +func TestUploadFile_FromBase64Content(t *testing.T) { + handler := http.NewServeMux() + handler.HandleFunc("POST /edge/api/rest/v2/upload", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("cert://test-uuid/client-cert.pem")) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Filename: "client-cert.pem", + ContentBase64: base64.StdEncoding.EncodeToString([]byte("fake cert bytes")), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.URI != "cert://test-uuid/client-cert.pem" { + t.Fatalf("unexpected uri: %s", output.URI) + } +} + +func TestUploadFile_RequiresContentBase64(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Filename: "x"}) + if err == nil { + t.Fatalf("expected an error when contentBase64 is not provided") + } +} + +func TestUploadFile_RequiresFilename(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ContentBase64: "Zm9v"}) + if err == nil { + t.Fatalf("expected an error when filename is not provided") + } +} + +func TestUploadFile_WithFilePath_FromDisk(t *testing.T) { + tmpFile, err := os.CreateTemp(t.TempDir(), "driver-*.jar") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + if _, err := tmpFile.Write([]byte("fake jar bytes")); err != nil { + t.Fatalf("failed to write temp file: %v", err) + } + if err := tmpFile.Close(); err != nil { + t.Fatalf("failed to close temp file: %v", err) + } + + handler := http.NewServeMux() + handler.HandleFunc("POST /edge/api/rest/v2/upload", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("jar://test-uuid/driver.jar")) + }) + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewToolWithFilePath(client).Handler(t.Context(), tools.InputWithFilePath{ + Filename: "driver.jar", + FilePath: tmpFile.Name(), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.URI != "jar://test-uuid/driver.jar" { + t.Fatalf("unexpected uri: %s", output.URI) + } +} + +func TestUploadFile_WithFilePath_RequiresExactlyOneSource(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + + _, err := tools.NewToolWithFilePath(client).Handler(t.Context(), tools.InputWithFilePath{Filename: "x"}) + if err == nil { + t.Fatalf("expected an error when neither contentBase64 nor filePath is provided") + } + + _, err = tools.NewToolWithFilePath(client).Handler(t.Context(), tools.InputWithFilePath{ + Filename: "x", + ContentBase64: "Zm9v", + FilePath: "/tmp/x", + }) + if err == nil { + t.Fatalf("expected an error when both contentBase64 and filePath are provided") + } +} + +func TestUploadFile_WithFilePath_ReadError(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + + output, err := tools.NewToolWithFilePath(client).Handler(t.Context(), tools.InputWithFilePath{ + Filename: "driver.jar", + FilePath: "/nonexistent/path/driver.jar", + }) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure for a nonexistent file") + } +}