From ac8062e2a50213c474c5b5cfda31638816accdd1 Mon Sep 17 00:00:00 2001 From: Dary Schaeken Date: Mon, 6 Jul 2026 09:18:45 +0200 Subject: [PATCH 01/10] feat: add jdbc-ingestion lifecycle MCP tools (connections, capabilities, database config, ingestion) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 14 new tools covering the full flow for DEV-202506 (epic DEV-196736): edge site/capability-type discovery, community/domain scaffolding, connection and capability creation (with driver/file upload and open-ended vendor property support), connection testing, database registration and schema sync configuration, ingestion triggering, and job status polling. Also adds get_data_source_setup_guide, which looks up driver class, connection string format, and per-auth-method connection properties for any of Collibra's ~68 supported data sources by parsing Collibra's public documentation site — avoiding the need to hand-encode each connector's auth requirements. --- go.mod | 5 +- go.sum | 14 +- pkg/clients/catalog_database_client.go | 244 ++++++++++++ pkg/clients/data_source_docs_client.go | 356 ++++++++++++++++++ pkg/clients/data_source_docs_client_test.go | 150 ++++++++ pkg/clients/dgc_community_client.go | 96 +++++ pkg/clients/edge_capability_client.go | 87 +++++ pkg/clients/edge_connection_client.go | 145 +++++++ pkg/clients/edge_job_client.go | 58 +++ pkg/clients/edge_site_client.go | 99 +++++ pkg/tools/configure_database/tool.go | 200 ++++++++++ pkg/tools/configure_database/tool_test.go | 145 +++++++ pkg/tools/create_capability/tool.go | 78 ++++ pkg/tools/create_capability/tool_test.go | 95 +++++ pkg/tools/create_community/tool.go | 49 +++ pkg/tools/create_community/tool_test.go | 61 +++ pkg/tools/create_connection/tool.go | 164 ++++++++ pkg/tools/create_connection/tool_test.go | 231 ++++++++++++ pkg/tools/create_domain/tool.go | 63 ++++ pkg/tools/create_domain/tool_test.go | 62 +++ pkg/tools/get_data_source_setup_guide/tool.go | 124 ++++++ .../get_data_source_setup_guide/tool_test.go | 139 +++++++ pkg/tools/get_job_status/tool.go | 49 +++ pkg/tools/get_job_status/tool_test.go | 46 +++ pkg/tools/list_capability_types/tool.go | 54 +++ pkg/tools/list_capability_types/tool_test.go | 47 +++ pkg/tools/list_edge_sites/tool.go | 40 ++ pkg/tools/list_edge_sites/tool_test.go | 38 ++ pkg/tools/register.go | 24 ++ pkg/tools/start_ingestion/tool.go | 54 +++ pkg/tools/start_ingestion/tool_test.go | 103 +++++ pkg/tools/test_connection/tool.go | 57 +++ pkg/tools/test_connection/tool_test.go | 73 ++++ pkg/tools/upload_file/tool.go | 95 +++++ pkg/tools/upload_file/tool_test.go | 84 +++++ 35 files changed, 3421 insertions(+), 8 deletions(-) create mode 100644 pkg/clients/catalog_database_client.go create mode 100644 pkg/clients/data_source_docs_client.go create mode 100644 pkg/clients/data_source_docs_client_test.go create mode 100644 pkg/clients/dgc_community_client.go create mode 100644 pkg/clients/edge_capability_client.go create mode 100644 pkg/clients/edge_connection_client.go create mode 100644 pkg/clients/edge_job_client.go create mode 100644 pkg/clients/edge_site_client.go create mode 100644 pkg/tools/configure_database/tool.go create mode 100644 pkg/tools/configure_database/tool_test.go create mode 100644 pkg/tools/create_capability/tool.go create mode 100644 pkg/tools/create_capability/tool_test.go create mode 100644 pkg/tools/create_community/tool.go create mode 100644 pkg/tools/create_community/tool_test.go create mode 100644 pkg/tools/create_connection/tool.go create mode 100644 pkg/tools/create_connection/tool_test.go create mode 100644 pkg/tools/create_domain/tool.go create mode 100644 pkg/tools/create_domain/tool_test.go create mode 100644 pkg/tools/get_data_source_setup_guide/tool.go create mode 100644 pkg/tools/get_data_source_setup_guide/tool_test.go create mode 100644 pkg/tools/get_job_status/tool.go create mode 100644 pkg/tools/get_job_status/tool_test.go create mode 100644 pkg/tools/list_capability_types/tool.go create mode 100644 pkg/tools/list_capability_types/tool_test.go create mode 100644 pkg/tools/list_edge_sites/tool.go create mode 100644 pkg/tools/list_edge_sites/tool_test.go create mode 100644 pkg/tools/start_ingestion/tool.go create mode 100644 pkg/tools/start_ingestion/tool_test.go create mode 100644 pkg/tools/test_connection/tool.go create mode 100644 pkg/tools/test_connection/tool_test.go create mode 100644 pkg/tools/upload_file/tool.go create mode 100644 pkg/tools/upload_file/tool_test.go diff --git a/go.mod b/go.mod index 50af924..666a8db 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/yuin/goldmark v1.8.2 + golang.org/x/net v0.56.0 ) require ( @@ -26,7 +27,7 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 81fa5d8..155aa8d 100644 --- a/go.sum +++ b/go.sum @@ -56,14 +56,16 @@ github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= -golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= -golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/clients/catalog_database_client.go b/pkg/clients/catalog_database_client.go new file mode 100644 index 0000000..81c3010 --- /dev/null +++ b/pkg/clients/catalog_database_client.go @@ -0,0 +1,244 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +const catalogDatabaseBasePath = "/rest/catalogDatabase/v1" + +// DatabaseConnection is a discovered connection to a specific database (catalog) on a +// data source, returned by GET /databaseConnections. +type DatabaseConnection struct { + ID string `json:"id"` + Name string `json:"name"` + EdgeConnectionID string `json:"edgeConnectionId"` + DatabaseID string `json:"databaseId,omitempty"` +} + +// SchemaConnection is a discovered connection to a specific schema within a database, +// returned by GET /schemaConnections. +type SchemaConnection struct { + ID string `json:"id"` + Name string `json:"name"` + DatabaseConnectionID string `json:"databaseConnectionId"` + SchemaID string `json:"schemaId,omitempty"` +} + +// Database is a registered Database asset, returned by POST/GET /databases. +type Database struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + CommunityID string `json:"communityId"` + OwnerIDs []string `json:"ownerIds"` + ParentSystemID string `json:"parentSystemId"` + DatabaseConnectionID string `json:"databaseConnectionId"` + EdgeConnectionStatus string `json:"edgeConnectionStatus,omitempty"` +} + +// AddDatabaseRequest is the request body for POST /databases. +type AddDatabaseRequest struct { + DatabaseConnectionID string `json:"databaseConnectionId"` + CommunityID string `json:"communityId"` + ParentSystemID string `json:"parentSystemId"` + OwnerIDs []string `json:"ownerIds"` + Description string `json:"description,omitempty"` +} + +// MetadataSynchronizationRule defines which tables of a schema get synchronized. +type MetadataSynchronizationRule struct { + Include string `json:"include,omitempty"` + Exclude string `json:"exclude,omitempty"` + TargetDomainID string `json:"targetDomainId,omitempty"` + SkipViews bool `json:"skipViews,omitempty"` +} + +// SchemaMetadataConfiguration links a schema connection to the synchronization rules +// that decide what gets ingested for it. +type SchemaMetadataConfiguration struct { + SchemaConnectionID string `json:"schemaConnectionId"` + SynchronizationRules []MetadataSynchronizationRule `json:"synchronizationRules,omitempty"` +} + +// Job represents an asynchronous job triggered by the database registration API. +type Job struct { + ID string `json:"id"` + State string `json:"state,omitempty"` + Result string `json:"result,omitempty"` + Message string `json:"message,omitempty"` +} + +// DatabaseMetadataSynchronizationRequest is the request body for +// POST /databases/{databaseId}/synchronizeMetadata. +type DatabaseMetadataSynchronizationRequest struct { + SchemaConnectionIDs []string `json:"schemaConnectionIds,omitempty"` +} + +type pagedResponse[T any] struct { + Results []T `json:"results"` +} + +// RefreshDatabaseConnections asynchronously refreshes the database connections known +// for an Edge connection via POST /databaseConnections/refresh?edgeConnectionId=... +// The refresh runs in the background; call ListDatabaseConnections afterwards (with +// retries) to see newly discovered connections. +func RefreshDatabaseConnections(ctx context.Context, client *http.Client, edgeConnectionID string) error { + endpoint := catalogDatabaseBasePath + "/databaseConnections/refresh?edgeConnectionId=" + edgeConnectionID + return doCatalogDatabasePost(ctx, client, endpoint, nil, http.StatusAccepted) +} + +// ListDatabaseConnections lists database connections discovered for an Edge connection +// via GET /databaseConnections?edgeConnectionId=... +func ListDatabaseConnections(ctx context.Context, client *http.Client, edgeConnectionID string) ([]DatabaseConnection, error) { + endpoint := catalogDatabaseBasePath + "/databaseConnections?edgeConnectionId=" + edgeConnectionID + var response pagedResponse[DatabaseConnection] + if err := doCatalogDatabaseGet(ctx, client, endpoint, &response); err != nil { + return nil, err + } + return response.Results, nil +} + +// RegisterDatabase creates a Database asset from a discovered database connection via +// POST /databases. +func RegisterDatabase(ctx context.Context, client *http.Client, request AddDatabaseRequest) (*Database, error) { + body, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("registering database: marshaling request: %w", err) + } + var database Database + if err := doCatalogDatabasePostJSON(ctx, client, catalogDatabaseBasePath+"/databases", body, http.StatusCreated, &database); err != nil { + return nil, err + } + return &database, nil +} + +// RefreshSchemaConnections asynchronously refreshes the schema connections known for a +// database connection via POST /schemaConnections/refresh?databaseConnectionId=... +func RefreshSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) error { + endpoint := catalogDatabaseBasePath + "/schemaConnections/refresh?databaseConnectionId=" + databaseConnectionID + return doCatalogDatabasePost(ctx, client, endpoint, nil, http.StatusAccepted) +} + +// ListSchemaConnections lists schema connections discovered for a database connection +// via GET /schemaConnections?databaseConnectionId=... +func ListSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) ([]SchemaConnection, error) { + endpoint := catalogDatabaseBasePath + "/schemaConnections?databaseConnectionId=" + databaseConnectionID + var response pagedResponse[SchemaConnection] + if err := doCatalogDatabaseGet(ctx, client, endpoint, &response); err != nil { + return nil, err + } + return response.Results, nil +} + +// SetSchemaMetadataConfigurationsBatch defines synchronization rules for one or more +// schema connections via POST /schemaMetadataConfigurations/batch. +func SetSchemaMetadataConfigurationsBatch(ctx context.Context, client *http.Client, configurations []SchemaMetadataConfiguration) ([]SchemaMetadataConfiguration, error) { + body, err := json.Marshal(configurations) + if err != nil { + return nil, fmt.Errorf("setting schema metadata configurations: marshaling request: %w", err) + } + var result []SchemaMetadataConfiguration + if err := doCatalogDatabasePostJSON(ctx, client, catalogDatabaseBasePath+"/schemaMetadataConfigurations/batch", body, http.StatusCreated, &result); err != nil { + return nil, err + } + return result, nil +} + +// SynchronizeDatabaseMetadata starts the jdbc-ingestion capability run for a registered +// Database asset via POST /databases/{databaseId}/synchronizeMetadata. An empty +// schemaConnectionIds list synchronizes all schemas that already have a +// SchemaMetadataConfiguration. +func SynchronizeDatabaseMetadata(ctx context.Context, client *http.Client, databaseID string, schemaConnectionIDs []string) (*Job, error) { + body, err := json.Marshal(DatabaseMetadataSynchronizationRequest{SchemaConnectionIDs: schemaConnectionIDs}) + if err != nil { + return nil, fmt.Errorf("starting ingestion: marshaling request: %w", err) + } + endpoint := catalogDatabaseBasePath + "/databases/" + databaseID + "/synchronizeMetadata" + var job Job + if err := doCatalogDatabasePostJSON(ctx, client, endpoint, body, http.StatusAccepted, &job); err != nil { + return nil, err + } + return &job, nil +} + +func doCatalogDatabaseGet(ctx context.Context, client *http.Client, endpoint string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("calling %s: building request: %w", endpoint, err) + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("calling %s: sending request: %w", endpoint, err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("calling %s: reading response: %w", endpoint, err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("calling %s: unexpected status %d: %s", endpoint, resp.StatusCode, string(body)) + } + + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("calling %s: decoding response: %w", endpoint, err) + } + + return nil +} + +func doCatalogDatabasePost(ctx context.Context, client *http.Client, endpoint string, body []byte, expectedStatus int) error { + return doCatalogDatabasePostJSON(ctx, client, endpoint, body, expectedStatus, nil) +} + +func doCatalogDatabasePostJSON(ctx context.Context, client *http.Client, endpoint string, body []byte, expectedStatus int, out any) error { + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, reader) + if err != nil { + return fmt.Errorf("calling %s: building request: %w", endpoint, err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("calling %s: sending request: %w", endpoint, err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("calling %s: reading response: %w", endpoint, err) + } + + if resp.StatusCode != expectedStatus { + switch resp.StatusCode { + case http.StatusConflict: + return fmt.Errorf("calling %s: conflict, an operation may already be in progress: %s", endpoint, string(respBody)) + default: + return fmt.Errorf("calling %s: unexpected status %d: %s", endpoint, resp.StatusCode, string(respBody)) + } + } + + if out == nil || len(respBody) == 0 { + return nil + } + + if err := json.Unmarshal(respBody, out); err != nil { + return fmt.Errorf("calling %s: decoding response: %w", endpoint, err) + } + + return nil +} diff --git a/pkg/clients/data_source_docs_client.go b/pkg/clients/data_source_docs_client.go new file mode 100644 index 0000000..d69e5d4 --- /dev/null +++ b/pkg/clients/data_source_docs_client.go @@ -0,0 +1,356 @@ +// Package clients: data_source_docs_client.go fetches connection setup guidance +// (driver class, JDBC connection string format, and per-auth-method connection +// properties) for any of Collibra's ~68 supported data sources, from Collibra's +// public product documentation site. +// +// IMPORTANT — this is a documented gap, not a stable integration: +// +// There is no published/versioned API for this. The URLs and HTML structure below +// were reverse-engineered by reading the JavaScript +// (productresources.collibra.com/docs/catalog-connectors/Content/Resources/JavaScripts/power-catcon.js) +// that Collibra's own docs site uses internally to render its "data source" selector +// widget. Collibra could restructure or remove these pages at any time without +// notice, since they were never intended to be consumed as an API. Treat every +// function here as best-effort: a fetch or parse failure means "fall back to asking +// the user for the connection properties directly," not a hard error condition to +// retry aggressively. +// +// If this breaks: re-derive the URL pattern by fetching +// https://productresources.collibra.com/docs/collibra/latest/Content/Edge/JDBCConnections/ta_create-jdbc-connection.htm?data-source= +// in a real browser with devtools open, and inspecting network requests fired by +// power-catcon.js's fetchCatConDocs() function. +package clients + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + + "golang.org/x/net/html" +) + +const catalogConnectorsBaseURL = "https://productresources.collibra.com/docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors" + +// DataSourceSlug identifies one of Collibra's supported data sources in the docs +// site's connector selector (e.g. {ID: "snowflake", Name: "Snowflake"}). +type DataSourceSlug struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// DataSourceInfo is driver/connection-string level documentation for a data source. +type DataSourceInfo struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + DriverClassName string `json:"driverClassName,omitempty"` + ConnectionStringFormat string `json:"connectionStringFormat,omitempty"` + MarketplaceURL string `json:"marketplaceUrl,omitempty"` + Prerequisites string `json:"prerequisites,omitempty"` +} + +// ConnectionPropertyDoc is one documented connection property row. +type ConnectionPropertyDoc struct { + Required bool `json:"required"` + Name string `json:"name"` + Type string `json:"type"` + ValueType string `json:"valueType"` + Description string `json:"description,omitempty"` +} + +// AuthMethodProperties groups the connection properties documented for one +// authentication method (e.g. "Key pair (unencrypted)", "OAuth"). +type AuthMethodProperties struct { + Name string `json:"name"` + Properties []ConnectionPropertyDoc `json:"properties"` +} + +// ListDataSources fetches the full list of data sources shown in Collibra's docs +// connector selector (68 as of 2026-07-04). +func ListDataSources(ctx context.Context, client *http.Client) ([]DataSourceSlug, error) { + doc, err := fetchAndParseHTML(ctx, client, catalogConnectorsBaseURL+"/_all-data-sources/selector/selector.htm") + if err != nil { + return nil, fmt.Errorf("listing data sources: %w", err) + } + + var sources []DataSourceSlug + seen := map[string]bool{} + for _, div := range htmlFindAll(doc, isElementWithClass("div", "catalogConnector")) { + id := htmlAttr(div, "id") + if id == "" || seen[id] { + continue + } + seen[id] = true + sources = append(sources, DataSourceSlug{ID: id, Name: strings.TrimSpace(htmlText(div))}) + } + + if len(sources) == 0 { + return nil, fmt.Errorf("listing data sources: no connectors found — the docs site's page structure may have changed") + } + return sources, nil +} + +// GetDataSourceInfo fetches driver class name, JDBC connection string format, +// Marketplace listing URL, and prerequisites for a data source, by its slug (as +// returned by ListDataSources). +func GetDataSourceInfo(ctx context.Context, client *http.Client, slug string) (*DataSourceInfo, error) { + doc, err := fetchAndParseHTML(ctx, client, catalogConnectorsBaseURL+"/"+slug+"/data-source-information.htm") + if err != nil { + return nil, fmt.Errorf("getting data source info for %q: %w", slug, err) + } + + info := &DataSourceInfo{ID: slug} + + if el := htmlFindByID(doc, "catcon-driver"); el != nil { + info.DriverClassName = strings.TrimSpace(htmlText(el)) + } + if el := htmlFindByID(doc, "catcon-jdbc-url"); el != nil { + if code := htmlFind(el, isElement("code")); code != nil { + info.ConnectionStringFormat = strings.TrimSpace(htmlText(code)) + } + } + if el := htmlFindByID(doc, "catcon-marketplace"); el != nil { + if a := htmlFind(el, isElement("a")); a != nil { + info.MarketplaceURL = htmlAttr(a, "href") + } + } + if el := htmlFindByID(doc, "catcon-prerequisites"); el != nil { + info.Prerequisites = collapseWhitespace(htmlText(el)) + } + + if info.DriverClassName == "" && info.ConnectionStringFormat == "" { + return nil, fmt.Errorf("getting data source info for %q: no recognizable content found — the docs site's page structure may have changed, or %q is not a valid slug", slug, slug) + } + + return info, nil +} + +// GetConnectionProperties fetches the documented connection properties for a data +// source, grouped by authentication method (e.g. a Snowflake connection documents +// separate property sets for "Username and password", "Key pair (unencrypted)", +// "OAuth", etc.). Use this to know exactly which named properties to pass as +// create_connection's additionalProperties for a given auth method. +func GetConnectionProperties(ctx context.Context, client *http.Client, slug string) ([]AuthMethodProperties, error) { + doc, err := fetchAndParseHTML(ctx, client, catalogConnectorsBaseURL+"/"+slug+"/minimum-connection-properties-edge.htm") + if err != nil { + return nil, fmt.Errorf("getting connection properties for %q: %w", slug, err) + } + + const subselectorPrefix = "data-sources/catalog-connector-subselectors." + + // Map auth-method key (e.g. "key-pair-unencrypted") to its human-readable name, + // read from the radio button selector's and + // sibling NAME. + names := map[string]string{} + for _, input := range htmlFindAll(doc, isElement("input")) { + value := htmlAttr(input, "value") + if !strings.HasPrefix(value, subselectorPrefix) { + continue + } + key := strings.TrimPrefix(value, subselectorPrefix) + if label := input.Parent; label != nil { + if span := htmlFind(label, isElement("span")); span != nil { + names[key] = strings.TrimSpace(htmlText(span)) + } + } + } + + var methods []AuthMethodProperties + seen := map[string]bool{} + for _, div := range htmlFindAll(doc, isElement("div")) { + cond := htmlAttr(div, "data-mc-conditions") + if cond == "" { + continue + } + + var key string + for _, tok := range strings.Split(cond, ",") { + tok = strings.TrimSpace(tok) + if k, ok := strings.CutPrefix(tok, subselectorPrefix); ok && k != "all" { + key = k + break + } + } + if key == "" || seen[key] { + continue + } + + table := htmlFind(div, isElement("table")) + if table == nil { + continue + } + thead := htmlFind(table, isElement("thead")) + if thead == nil || !strings.Contains(strings.ToLower(htmlText(thead)), "required") { + // Not a connection-properties table (this div's condition may gate some + // other kind of content) — skip rather than misparse it. + continue + } + + seen[key] = true + name := names[key] + if name == "" { + name = key + } + methods = append(methods, AuthMethodProperties{Name: name, Properties: parsePropertyRows(table)}) + } + + if len(methods) == 0 { + return nil, fmt.Errorf("getting connection properties for %q: no auth-method property tables found — the docs site's page structure may have changed, or this data source may not document any (some sources need no extra properties beyond the driver/connection string)", slug) + } + + return methods, nil +} + +func parsePropertyRows(table *html.Node) []ConnectionPropertyDoc { + tbody := htmlFind(table, isElement("tbody")) + if tbody == nil { + return nil + } + + var props []ConnectionPropertyDoc + for _, tr := range directChildren(tbody, "tr") { + tds := directChildren(tr, "td") + if len(tds) < 5 { + continue + } + + required := false + if img := htmlFind(tds[0], isElement("img")); img != nil { + required = strings.EqualFold(htmlAttr(img, "alt"), "yes") + } else { + required = strings.EqualFold(strings.TrimSpace(htmlText(tds[0])), "yes") + } + + props = append(props, ConnectionPropertyDoc{ + Required: required, + Name: strings.TrimSpace(htmlText(tds[1])), + Type: strings.TrimSpace(htmlText(tds[2])), + ValueType: strings.TrimSpace(htmlText(tds[3])), + Description: collapseWhitespace(htmlText(tds[4])), + }) + } + return props +} + +func fetchAndParseHTML(ctx context.Context, client *http.Client, url string) (*html.Node, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("building request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("not found (%s)", url) + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) + return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body)) + } + + doc, err := html.Parse(resp.Body) + if err != nil { + return nil, fmt.Errorf("parsing HTML: %w", err) + } + return doc, nil +} + +// --- Minimal HTML tree helpers (avoids pulling in a full CSS-selector library for +// what is otherwise a handful of narrow, structure-specific lookups). --- + +func htmlFindAll(n *html.Node, match func(*html.Node) bool) []*html.Node { + var out []*html.Node + var walk func(*html.Node) + walk = func(n *html.Node) { + if match(n) { + out = append(out, n) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return out +} + +func htmlFind(n *html.Node, match func(*html.Node) bool) *html.Node { + if match(n) { + return n + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if found := htmlFind(c, match); found != nil { + return found + } + } + return nil +} + +func htmlFindByID(n *html.Node, id string) *html.Node { + return htmlFind(n, func(n *html.Node) bool { + return n.Type == html.ElementNode && htmlAttr(n, "id") == id + }) +} + +func isElement(tag string) func(*html.Node) bool { + return func(n *html.Node) bool { return n.Type == html.ElementNode && n.Data == tag } +} + +func isElementWithClass(tag, class string) func(*html.Node) bool { + return func(n *html.Node) bool { + if n.Type != html.ElementNode || n.Data != tag { + return false + } + for _, c := range strings.Fields(htmlAttr(n, "class")) { + if c == class { + return true + } + } + return false + } +} + +// directChildren returns n's immediate element children matching tag (not a +// recursive search) — used for table row/cell parsing where a cell's own content +// (e.g. a collapsible "steps" dropdown) could otherwise be mistaken for sibling rows. +func directChildren(n *html.Node, tag string) []*html.Node { + var out []*html.Node + for c := n.FirstChild; c != nil; c = c.NextSibling { + if c.Type == html.ElementNode && c.Data == tag { + out = append(out, c) + } + } + return out +} + +func htmlAttr(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +func htmlText(n *html.Node) string { + var sb strings.Builder + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.TextNode { + sb.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return sb.String() +} + +func collapseWhitespace(s string) string { + return strings.Join(strings.Fields(s), " ") +} diff --git a/pkg/clients/data_source_docs_client_test.go b/pkg/clients/data_source_docs_client_test.go new file mode 100644 index 0000000..a19f048 --- /dev/null +++ b/pkg/clients/data_source_docs_client_test.go @@ -0,0 +1,150 @@ +package clients_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" +) + +const selectorFixture = ` + +` + +const dataSourceInfoFixture = ` +
+

The JDBC connection string:

+

jdbc:snowflake://<accountname>.snowflakecomputing.com

+

The JDBC driver class name:

+
net.snowflake.client.jdbc.SnowflakeDriver
+

The Collibra Marketplace listing URL:

+
link
+

The connection prerequisites:

+
  • Download the driver from Marketplace.
    • +
    +` + +const connectionPropertiesFixture = ` +
    + + +
    +
    +

    To establish a connection using username and password authentication.

    + + + + +

    Required?

    Name

    Type

    Value Type

    Value

    YesPasswordTextSecretThe password.
    NoroleTextPlaintextThe role.
    +
    +
    + + + +

    Required?

    Name

    Type

    Value Type

    Value

    Yesprivate_key_fileFilePlaintextThe key file.
    +
    +` + +func newDocsTestServer(t *testing.T) *http.Client { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/_all-data-sources/selector/selector.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(selectorFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/snowflake/data-source-information.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(dataSourceInfoFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/snowflake/minimum-connection-properties-edge.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(connectionPropertiesFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/does-not-exist/data-source-information.htm", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + return testutil.NewClient(httptest.NewServer(mux)) +} + +func TestListDataSources(t *testing.T) { + client := newDocsTestServer(t) + sources, err := clients.ListDataSources(t.Context(), client) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(sources) != 2 { + t.Fatalf("expected 2 data sources (hidden placeholder excluded), got %d: %+v", len(sources), sources) + } + if sources[0].ID != "snowflake" || sources[0].Name != "Snowflake" { + t.Fatalf("unexpected first source: %+v", sources[0]) + } +} + +func TestGetDataSourceInfo(t *testing.T) { + client := newDocsTestServer(t) + info, err := clients.GetDataSourceInfo(t.Context(), client, "snowflake") + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if info.DriverClassName != "net.snowflake.client.jdbc.SnowflakeDriver" { + t.Fatalf("unexpected driver class: %s", info.DriverClassName) + } + if info.ConnectionStringFormat != "jdbc:snowflake://.snowflakecomputing.com" { + t.Fatalf("unexpected connection string: %s", info.ConnectionStringFormat) + } + if info.MarketplaceURL != "https://marketplace.collibra.com/listings/snowflake" { + t.Fatalf("unexpected marketplace url: %s", info.MarketplaceURL) + } + if info.Prerequisites == "" { + t.Fatalf("expected prerequisites to be populated") + } +} + +func TestGetDataSourceInfo_NotFound(t *testing.T) { + client := newDocsTestServer(t) + _, err := clients.GetDataSourceInfo(t.Context(), client, "does-not-exist") + if err == nil { + t.Fatalf("expected an error for a 404") + } +} + +func TestGetConnectionProperties(t *testing.T) { + client := newDocsTestServer(t) + methods, err := clients.GetConnectionProperties(t.Context(), client, "snowflake") + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(methods) != 2 { + t.Fatalf("expected 2 auth methods, got %d: %+v", len(methods), methods) + } + + byName := map[string]clients.AuthMethodProperties{} + for _, m := range methods { + byName[m.Name] = m + } + + userPass, ok := byName["Username and password"] + if !ok { + t.Fatalf("expected 'Username and password' auth method, got: %+v", methods) + } + if len(userPass.Properties) != 2 { + t.Fatalf("expected 2 properties, got: %+v", userPass.Properties) + } + if !userPass.Properties[0].Required || userPass.Properties[0].Name != "Password" || userPass.Properties[0].ValueType != "Secret" { + t.Fatalf("unexpected Password property: %+v", userPass.Properties[0]) + } + if userPass.Properties[1].Required { + t.Fatalf("expected role to be not required: %+v", userPass.Properties[1]) + } + + keyPair, ok := byName["Key pair (encrypted)"] + if !ok { + t.Fatalf("expected 'Key pair (encrypted)' auth method, got: %+v", methods) + } + if len(keyPair.Properties) != 1 || keyPair.Properties[0].Name != "private_key_file" || keyPair.Properties[0].Type != "File" { + t.Fatalf("unexpected key pair properties: %+v", keyPair.Properties) + } +} diff --git a/pkg/clients/dgc_community_client.go b/pkg/clients/dgc_community_client.go new file mode 100644 index 0000000..472e521 --- /dev/null +++ b/pkg/clients/dgc_community_client.go @@ -0,0 +1,96 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// CommunityDetails is the response shape returned by POST /rest/2.0/communities. +type CommunityDetails struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` +} + +// CreateCommunityRequest is the request body for POST /rest/2.0/communities. +type CreateCommunityRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` +} + +// CreateCommunity creates a new community via POST /rest/2.0/communities. +func CreateCommunity(ctx context.Context, client *http.Client, request CreateCommunityRequest) (*CommunityDetails, error) { + var community CommunityDetails + if err := postJSON(ctx, client, "/rest/2.0/communities", request, http.StatusCreated, &community); err != nil { + return nil, fmt.Errorf("creating community: %w", err) + } + return &community, nil +} + +// DomainDetails is the response shape returned by POST /rest/2.0/domains. +type DomainDetails struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + CommunityID string `json:"communityId"` + TypeID string `json:"typeId"` +} + +// CreateDomainRequest is the request body for POST /rest/2.0/domains. +type CreateDomainRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + CommunityID string `json:"communityId"` + TypeID string `json:"typeId"` + ExcludedFromAutoHyperlinking bool `json:"excludedFromAutoHyperlinking,omitempty"` +} + +// CreateDomain creates a new domain via POST /rest/2.0/domains. +func CreateDomain(ctx context.Context, client *http.Client, request CreateDomainRequest) (*DomainDetails, error) { + var domain DomainDetails + if err := postJSON(ctx, client, "/rest/2.0/domains", request, http.StatusCreated, &domain); err != nil { + return nil, fmt.Errorf("creating domain: %w", err) + } + return &domain, nil +} + +// postJSON is a small shared helper for simple "POST body, decode JSON response" +// calls against DGC's rest/2.0 API family. +func postJSON(ctx context.Context, client *http.Client, endpoint string, request any, expectedStatus int, out any) error { + body, err := json.Marshal(request) + if err != nil { + return fmt.Errorf("marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading response: %w", err) + } + + if resp.StatusCode != expectedStatus { + return fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + + if err := json.Unmarshal(respBody, out); err != nil { + return fmt.Errorf("decoding response: %w", err) + } + + return nil +} diff --git a/pkg/clients/edge_capability_client.go b/pkg/clients/edge_capability_client.go new file mode 100644 index 0000000..54be125 --- /dev/null +++ b/pkg/clients/edge_capability_client.go @@ -0,0 +1,87 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// Capability is the response shape returned by the Edge capability management API. +type Capability struct { + ID string `json:"id"` + Type any `json:"type,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + EdgeSiteID string `json:"edgeSiteId"` + EdgeSiteName string `json:"edgeSiteName,omitempty"` + Parameters map[string]any `json:"parameters"` +} + +// CapabilityRequest is the request body for creating or updating a capability via +// POST /edge/api/rest/v2/capabilities or PUT /edge/api/rest/v2/capabilities/{id}. +type CapabilityRequest struct { + TypeID string `json:"typeId"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + EdgeSiteID string `json:"edgeSiteId"` + Parameters map[string]any `json:"parameters"` +} + +// CreateCapability creates a new Edge capability via POST /edge/api/rest/v2/capabilities. +// The server assigns the capability id. +func CreateCapability(ctx context.Context, client *http.Client, request CapabilityRequest) (*Capability, error) { + return doCapabilityRequest(ctx, client, http.MethodPost, "/edge/api/rest/v2/capabilities", request) +} + +// CreateOrUpdateCapability creates or updates an Edge capability with a known id via +// PUT /edge/api/rest/v2/capabilities/{id}. +func CreateOrUpdateCapability(ctx context.Context, client *http.Client, capabilityID string, request CapabilityRequest) (*Capability, error) { + endpoint := "/edge/api/rest/v2/capabilities/" + capabilityID + return doCapabilityRequest(ctx, client, http.MethodPut, endpoint, request) +} + +func doCapabilityRequest(ctx context.Context, client *http.Client, method, endpoint string, request CapabilityRequest) (*Capability, error) { + body, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("saving capability: marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("saving capability: building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("saving capability: sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("saving capability: reading response: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + switch resp.StatusCode { + case http.StatusBadRequest: + return nil, fmt.Errorf("saving capability: bad request (invalid parameters or typeId): %s", string(respBody)) + case http.StatusNotFound: + return nil, fmt.Errorf("saving capability: edge site not found: %s", string(respBody)) + default: + return nil, fmt.Errorf("saving capability: unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + } + + var result Capability + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("saving capability: decoding response: %w", err) + } + + return &result, nil +} diff --git a/pkg/clients/edge_connection_client.go b/pkg/clients/edge_connection_client.go new file mode 100644 index 0000000..4ea4346 --- /dev/null +++ b/pkg/clients/edge_connection_client.go @@ -0,0 +1,145 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" +) + +// Connection is the response shape returned by the Edge connection management API. +type Connection struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + TypeID string `json:"typeId,omitempty"` + EdgeSiteID string `json:"edgeSiteId"` + VaultID string `json:"vaultId,omitempty"` + Parameters map[string]any `json:"parameters"` +} + +// ConnectionRequest is the request body for creating or updating a connection via +// POST /edge/api/rest/v2/connections or PUT /edge/api/rest/v2/connections/{id}. +type ConnectionRequest struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + TypeID string `json:"typeId"` + EdgeSiteID string `json:"edgeSiteId"` + VaultID string `json:"vaultId,omitempty"` + Parameters map[string]any `json:"parameters"` +} + +// CreateConnection creates a new Edge connection via POST /edge/api/rest/v2/connections. +// The server assigns the connection id. +func CreateConnection(ctx context.Context, client *http.Client, request ConnectionRequest) (*Connection, error) { + return doConnectionRequest(ctx, client, http.MethodPost, "/edge/api/rest/v2/connections", request) +} + +// CreateOrUpdateConnection creates or updates an Edge connection with a known id via +// PUT /edge/api/rest/v2/connections/{connectionId}. +func CreateOrUpdateConnection(ctx context.Context, client *http.Client, connectionID string, request ConnectionRequest) (*Connection, error) { + endpoint := "/edge/api/rest/v2/connections/" + connectionID + return doConnectionRequest(ctx, client, http.MethodPut, endpoint, request) +} + +func doConnectionRequest(ctx context.Context, client *http.Client, method, endpoint string, request ConnectionRequest) (*Connection, error) { + body, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("saving connection: marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("saving connection: building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("saving connection: sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("saving connection: reading response: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + switch resp.StatusCode { + case http.StatusBadRequest: + return nil, fmt.Errorf("saving connection: bad request (invalid parameters or typeId): %s", string(respBody)) + case http.StatusNotFound: + return nil, fmt.Errorf("saving connection: edge site not found: %s", string(respBody)) + default: + return nil, fmt.Errorf("saving connection: unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + } + + var result Connection + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("saving connection: decoding response: %w", err) + } + + return &result, nil +} + +// UploadFile uploads an arbitrary file via POST /edge/api/rest/v2/upload and returns +// the artifact URI (e.g. "jar:///") to reference from any FILE-type +// connection or capability parameter — confirmed (via ConnectionPropertyDeserializer) +// that the parameter value is the bare artifact URI string, not a wrapped object; this +// covers JDBC driver jars as well as certs, keytabs, private keys, etc. +func UploadFile(ctx context.Context, client *http.Client, filename string, content []byte) (string, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, err := writer.CreateFormFile("file", filename) + if err != nil { + return "", fmt.Errorf("uploading file: creating form file: %w", err) + } + if _, err := part.Write(content); err != nil { + return "", fmt.Errorf("uploading file: writing content: %w", err) + } + if err := writer.Close(); err != nil { + return "", fmt.Errorf("uploading file: closing multipart writer: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/edge/api/rest/v2/upload", &body) + if err != nil { + return "", fmt.Errorf("uploading file: building request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + // This endpoint responds with produces=TEXT_PLAIN regardless of the request's + // Accept header's preference — asking for application/json gets a 406. + req.Header.Set("Accept", "text/plain") + + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("uploading file: sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("uploading file: reading response: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("uploading file: unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + + // The upload endpoint returns a plain-text artifact URI, but tolerate a + // JSON-quoted string too in case that ever changes. + var uri string + if err := json.Unmarshal(respBody, &uri); err != nil { + uri = string(bytes.Trim(respBody, "\" \n\t")) + } + if uri == "" { + return "", fmt.Errorf("uploading file: empty artifact URI in response: %s", string(respBody)) + } + + return uri, nil +} diff --git a/pkg/clients/edge_job_client.go b/pkg/clients/edge_job_client.go new file mode 100644 index 0000000..b899f12 --- /dev/null +++ b/pkg/clients/edge_job_client.go @@ -0,0 +1,58 @@ +package clients + +import ( + "context" + "fmt" + "net/http" + "strconv" +) + +// TestConnectionResponse is the response shape returned by +// GET /edge/api/rest/v2/connections/{connectionId}/test. +type TestConnectionResponse struct { + JobID string `json:"jobId"` + Success bool `json:"success"` + Message string `json:"message,omitempty"` +} + +// TestConnection tests an Edge connection via +// GET /edge/api/rest/v2/connections/{connectionId}/test?timeoutSec=. When timeoutSec +// is nil (or <= 0), the call returns immediately with a jobId while the test runs in +// the background — poll its result with GetJobStatusLog. When timeoutSec is positive, +// the call blocks up to that many seconds and returns the final result directly. +func TestConnection(ctx context.Context, client *http.Client, connectionID string, timeoutSec *int) (*TestConnectionResponse, error) { + endpoint := "/edge/api/rest/v2/connections/" + connectionID + "/test" + if timeoutSec != nil && *timeoutSec > 0 { + endpoint += "?timeoutSec=" + strconv.Itoa(*timeoutSec) + } + + var result TestConnectionResponse + if err := getJSON(ctx, client, endpoint, "testing connection", &result); err != nil { + return nil, err + } + return &result, nil +} + +// JobStatusLog is the response shape returned by GET /edge/api/rest/v2/jobs/{id}/statusLog. +type JobStatusLog struct { + JobID string `json:"jobId"` + Status string `json:"status"` + Message string `json:"message,omitempty"` + LastUpdatedDateTime string `json:"lastUpdatedDateTime,omitempty"` +} + +// GetJobStatusLog fetches the current status of an Edge job (e.g. one started by +// TestConnection or start_ingestion's SynchronizeDatabaseMetadata) via +// GET /edge/api/rest/v2/jobs/{id}/statusLog. Terminal statuses include SUCCEEDED, +// FAILED, CANCELLED, CAPABILITY_SUCCEEDED, and CAPABILITY_FAILED. +func GetJobStatusLog(ctx context.Context, client *http.Client, jobID string) (*JobStatusLog, error) { + endpoint := "/edge/api/rest/v2/jobs/" + jobID + "/statusLog" + var result JobStatusLog + if err := getJSON(ctx, client, endpoint, "getting job status", &result); err != nil { + return nil, err + } + if result.JobID == "" { + return nil, fmt.Errorf("getting job status: empty response for job %s", jobID) + } + return &result, nil +} diff --git a/pkg/clients/edge_site_client.go b/pkg/clients/edge_site_client.go new file mode 100644 index 0000000..25e2b8b --- /dev/null +++ b/pkg/clients/edge_site_client.go @@ -0,0 +1,99 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// EdgeSite is the response shape returned by the Edge site management API. +type EdgeSite struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + SiteType string `json:"siteType,omitempty"` + Status string `json:"status"` + VersionSyncStatus string `json:"versionSyncStatus,omitempty"` + InstalledVersion string `json:"installedVersion,omitempty"` + DesiredVersion string `json:"desiredVersion,omitempty"` +} + +// CapabilityType describes an available Edge capability type (e.g. "jdbc-ingestion") +// and the manifest that defines its expected install parameters. +type CapabilityType struct { + ID string `json:"id"` + // Manifest is arbitrary, capability-type-defined JSON. any (not json.RawMessage) + // is deliberate: json.RawMessage is a []byte under the hood, and chip's schema + // reflector doesn't know its custom marshaling, so it advertises an array/string + // schema for a value that's actually a JSON object at runtime. + Manifest any `json:"manifest,omitempty"` +} + +// ConnectionType describes an available Edge connection type and the manifest that +// defines its expected parameters. +type ConnectionType struct { + ID string `json:"id"` + Manifest any `json:"manifest,omitempty"` +} + +// GetEdgeSites lists all Edge sites via GET /edge/api/rest/v2/sites. +func GetEdgeSites(ctx context.Context, client *http.Client) ([]EdgeSite, error) { + var sites []EdgeSite + if err := getJSON(ctx, client, "/edge/api/rest/v2/sites", "listing edge sites", &sites); err != nil { + return nil, err + } + return sites, nil +} + +// GetCapabilityTypes lists the capability types available on an Edge site via +// GET /edge/api/rest/v2/sites/{siteId}/capabilityTypes. +func GetCapabilityTypes(ctx context.Context, client *http.Client, edgeSiteID string) ([]CapabilityType, error) { + var types []CapabilityType + endpoint := "/edge/api/rest/v2/sites/" + edgeSiteID + "/capabilityTypes" + if err := getJSON(ctx, client, endpoint, "listing capability types", &types); err != nil { + return nil, err + } + return types, nil +} + +// GetConnectionTypes lists the connection types available on an Edge site via +// GET /edge/api/rest/v2/sites/{siteId}/connectionTypes. +func GetConnectionTypes(ctx context.Context, client *http.Client, edgeSiteID string) ([]ConnectionType, error) { + var types []ConnectionType + endpoint := "/edge/api/rest/v2/sites/" + edgeSiteID + "/connectionTypes" + if err := getJSON(ctx, client, endpoint, "listing connection types", &types); err != nil { + return nil, err + } + return types, nil +} + +func getJSON(ctx context.Context, client *http.Client, endpoint, action string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("%s: building request: %w", action, err) + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("%s: sending request: %w", action, err) + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("%s: reading response: %w", action, err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s: unexpected status %d: %s", action, resp.StatusCode, string(body)) + } + + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("%s: decoding response: %w", action, err) + } + + return nil +} diff --git a/pkg/tools/configure_database/tool.go b/pkg/tools/configure_database/tool.go new file mode 100644 index 0000000..5e65546 --- /dev/null +++ b/pkg/tools/configure_database/tool.go @@ -0,0 +1,200 @@ +// Package configure_database implements the configure_database MCP tool: registers a +// database discovered through an Edge connection as a Database asset and configures +// which schemas/tables get synchronized, via the public catalogDatabase API. +// +// This wraps a 4-step flow (refresh database connections, register the database, +// refresh schema connections, set synchronization rules) as a single tool call. The +// refresh steps are asynchronous on the server, so this tool polls for their result +// with a bounded number of attempts; if a stage hasn't completed within that budget, +// it returns an error asking the agent to retry the tool call (the refresh calls are +// idempotent to re-issue). +package configure_database + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// Sleep is used for polling backoff. Overridable in tests to avoid slow test runs. +var Sleep = time.Sleep + +const ( + // pollAttempts * pollInterval bounds the wait for each async refresh. A + // first-time connection to a data source can take 30-60s to establish, so + // this budget needs enough headroom to cover that cold-start case. + pollAttempts = 14 + pollInterval = 5 * time.Second +) + +type Input struct { + EdgeConnectionID string `json:"edgeConnectionId" jsonschema:"UUID of the Edge connection (created via create_connection) to discover a database on."` + DatabaseName string `json:"databaseName,omitempty" jsonschema:"Optional. Exact name of the database (catalog) to register, as it appears at the data source. Required if the data source exposes more than one database/catalog through this connection; if there is exactly one, it is selected automatically."` + CommunityID string `json:"communityId" jsonschema:"UUID of the community the Database asset (and its automatically created domain) will be created in."` + ParentSystemID string `json:"parentSystemId" jsonschema:"UUID of the parent System asset the Database asset will be linked to."` + OwnerIDs []string `json:"ownerIds" jsonschema:"UUIDs of the users to assign as owners of the Database asset."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the Database asset."` + + Include string `json:"include,omitempty" jsonschema:"Optional. Comma-separated table name pattern to synchronize, '*' wildcard supported. Defaults to '*' (all tables)."` + Exclude string `json:"exclude,omitempty" jsonschema:"Optional. Comma-separated table name pattern to exclude from synchronization."` + TargetDomainID string `json:"targetDomainId,omitempty" jsonschema:"Optional. UUID of a domain to create synchronized assets in. If omitted, an automatically created domain per schema is used."` + SkipViews bool `json:"skipViews,omitempty" jsonschema:"Optional. If true, database views are excluded from synchronization."` +} + +type Output struct { + Database *clients.Database `json:"database,omitempty" jsonschema:"The registered Database asset."` + SchemaConnections []clients.SchemaConnection `json:"schemaConnections,omitempty" jsonschema:"The schema connections discovered under the database, now configured for synchronization."` + Success bool `json:"success" jsonschema:"Whether the database was fully configured."` + Error string `json:"error,omitempty" jsonschema:"Error message if configuration failed or is still in progress (safe to retry)."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "configure_database", + Title: "Configure Database for Ingestion", + Description: "Discovers a database through an Edge connection, registers it as a Database asset, and configures which tables get synchronized. Prerequisite for start_ingestion. Assumes the target community, and parent System asset already exist.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("edgeConnectionId", input.EdgeConnectionID); err != nil { + return Output{}, err + } + if err := validation.UUID("communityId", input.CommunityID); err != nil { + return Output{}, err + } + if err := validation.UUID("parentSystemId", input.ParentSystemID); err != nil { + return Output{}, err + } + if err := validation.UUIDs("ownerIds", input.OwnerIDs); err != nil { + return Output{}, err + } + if err := validation.UUIDOptional("targetDomainId", input.TargetDomainID); err != nil { + return Output{}, err + } + + databaseConnection, err := discoverDatabaseConnection(ctx, collibraClient, input.EdgeConnectionID, input.DatabaseName) + if err != nil { + return Output{Success: false, Error: err.Error()}, nil + } + + database, err := clients.RegisterDatabase(ctx, collibraClient, clients.AddDatabaseRequest{ + DatabaseConnectionID: databaseConnection.ID, + CommunityID: input.CommunityID, + ParentSystemID: input.ParentSystemID, + OwnerIDs: input.OwnerIDs, + Description: input.Description, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to register database: %s", err.Error())}, nil + } + + schemaConnections, err := discoverSchemaConnections(ctx, collibraClient, databaseConnection.ID) + if err != nil { + return Output{Database: database, Success: false, Error: err.Error()}, nil + } + + include := input.Include + if include == "" { + include = "*" + } + + configurations := make([]clients.SchemaMetadataConfiguration, len(schemaConnections)) + for i, schemaConnection := range schemaConnections { + configurations[i] = clients.SchemaMetadataConfiguration{ + SchemaConnectionID: schemaConnection.ID, + SynchronizationRules: []clients.MetadataSynchronizationRule{ + { + Include: include, + Exclude: input.Exclude, + TargetDomainID: input.TargetDomainID, + SkipViews: input.SkipViews, + }, + }, + } + } + + if _, err := clients.SetSchemaMetadataConfigurationsBatch(ctx, collibraClient, configurations); err != nil { + return Output{Database: database, SchemaConnections: schemaConnections, Success: false, + Error: fmt.Sprintf("failed to set schema synchronization rules: %s", err.Error())}, nil + } + + return Output{Database: database, SchemaConnections: schemaConnections, Success: true}, nil + } +} + +// discoverDatabaseConnection triggers a refresh of database connections for the given +// Edge connection and polls until at least one appears, then selects one by name (or +// the only one, if there's no ambiguity). +func discoverDatabaseConnection(ctx context.Context, client *http.Client, edgeConnectionID, databaseName string) (*clients.DatabaseConnection, error) { + if err := clients.RefreshDatabaseConnections(ctx, client, edgeConnectionID); err != nil { + return nil, fmt.Errorf("failed to refresh database connections: %w", err) + } + + var connections []clients.DatabaseConnection + for attempt := 0; attempt < pollAttempts; attempt++ { + if attempt > 0 { + Sleep(pollInterval) + } + found, err := clients.ListDatabaseConnections(ctx, client, edgeConnectionID) + if err != nil { + return nil, fmt.Errorf("failed to list database connections: %w", err) + } + if len(found) > 0 { + connections = found + break + } + } + + if len(connections) == 0 { + return nil, fmt.Errorf("no database connections were discovered for edge connection %s after %d attempts; the refresh may still be in progress — retry this tool call", edgeConnectionID, pollAttempts) + } + + if databaseName != "" { + for _, connection := range connections { + if connection.Name == databaseName { + return &connection, nil + } + } + return nil, fmt.Errorf("no database named %q found among discovered database connections for edge connection %s", databaseName, edgeConnectionID) + } + + if len(connections) > 1 { + return nil, fmt.Errorf("multiple databases discovered for edge connection %s; specify databaseName to select one", edgeConnectionID) + } + + return &connections[0], nil +} + +// discoverSchemaConnections triggers a refresh of schema connections for the given +// database connection and polls until at least one appears. +func discoverSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) ([]clients.SchemaConnection, error) { + if err := clients.RefreshSchemaConnections(ctx, client, databaseConnectionID); err != nil { + return nil, fmt.Errorf("failed to refresh schema connections: %w", err) + } + + for attempt := 0; attempt < pollAttempts; attempt++ { + if attempt > 0 { + Sleep(pollInterval) + } + found, err := clients.ListSchemaConnections(ctx, client, databaseConnectionID) + if err != nil { + return nil, fmt.Errorf("failed to list schema connections: %w", err) + } + if len(found) > 0 { + return found, nil + } + } + + return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts; the refresh may still be in progress — retry this tool call", databaseConnectionID, pollAttempts) +} diff --git a/pkg/tools/configure_database/tool_test.go b/pkg/tools/configure_database/tool_test.go new file mode 100644 index 0000000..9d57f17 --- /dev/null +++ b/pkg/tools/configure_database/tool_test.go @@ -0,0 +1,145 @@ +package configure_database_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/configure_database" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func init() { + tools.Sleep = func(time.Duration) {} +} + +func TestConfigureDatabase_Success(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + dbConnID, _ := uuid.NewUUID() + databaseID, _ := uuid.NewUUID() + schemaConnID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("edgeConnectionId") != edgeConnID.String() { + t.Fatalf("unexpected edgeConnectionId: %s", r.URL.Query().Get("edgeConnectionId")) + } + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.DatabaseConnection{ + {ID: dbConnID.String(), Name: "source", EdgeConnectionID: edgeConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { + if in.DatabaseConnectionID != dbConnID.String() { + t.Fatalf("unexpected databaseConnectionId: %s", in.DatabaseConnectionID) + } + return http.StatusCreated, clients.Database{ + ID: databaseID.String(), + Name: "source", + CommunityID: in.CommunityID, + OwnerIDs: in.OwnerIDs, + ParentSystemID: in.ParentSystemID, + DatabaseConnectionID: in.DatabaseConnectionID, + } + })) + mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("databaseConnectionId") != dbConnID.String() { + t.Fatalf("unexpected databaseConnectionId: %s", r.URL.Query().Get("databaseConnectionId")) + } + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/schemaConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.SchemaConnection{ + {ID: schemaConnID.String(), Name: "public", DatabaseConnectionID: dbConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/schemaMetadataConfigurations/batch", testutil.JsonHandlerInOut(func(r *http.Request, in []clients.SchemaMetadataConfiguration) (int, []clients.SchemaMetadataConfiguration) { + if len(in) != 1 || in[0].SchemaConnectionID != schemaConnID.String() { + t.Fatalf("unexpected batch payload: %+v", in) + } + if len(in[0].SynchronizationRules) != 1 || in[0].SynchronizationRules[0].Include != "*" { + t.Fatalf("unexpected synchronization rules: %+v", in[0].SynchronizationRules) + } + return http.StatusCreated, in + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Database.ID != databaseID.String() { + t.Fatalf("expected database id %s, got %s", databaseID.String(), output.Database.ID) + } + if len(output.SchemaConnections) != 1 { + t.Fatalf("expected 1 schema connection, got %d", len(output.SchemaConnections)) + } +} + +func TestConfigureDatabase_NoDatabaseConnectionsDiscovered(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{"results": []clients.DatabaseConnection{}} + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err != nil { + t.Fatalf("expected no error (failures reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when no database connections are discovered") + } + if output.Error == "" { + t.Fatalf("expected an error message") + } +} + +func TestConfigureDatabase_InvalidInput(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: "not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid edgeConnectionId") + } +} diff --git a/pkg/tools/create_capability/tool.go b/pkg/tools/create_capability/tool.go new file mode 100644 index 0000000..90d7ba2 --- /dev/null +++ b/pkg/tools/create_capability/tool.go @@ -0,0 +1,78 @@ +// Package create_capability implements the create_capability MCP tool: creates or +// updates an Edge capability (e.g. jdbc-ingestion or a lineage capability) via the +// private Edge capability management API. +package create_capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + CapabilityID string `json:"capabilityId,omitempty" jsonschema:"Optional. UUID of the capability to create or update. If provided, the capability is created with this exact id (or updated if it already exists). If omitted, a new capability is created and the server assigns an id."` + Name string `json:"name" jsonschema:"The capability name."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the capability."` + TypeID string `json:"typeId" jsonschema:"The id of the capability type (e.g. 'jdbc-ingestion'). Use the list_capability_types tool to discover available types and their expected parameters."` + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this capability will run. Use the list_edge_sites tool to discover available sites."` + Parameters map[string]any `json:"parameters" jsonschema:"Capability install parameters as defined by the capability type's manifest. For jdbc-ingestion, this includes 'connection' (the id of a connection created via create_connection), 'data-source-type', 'message-mode', and an optional 'other-settings' list of {name, type, value}."` +} + +type Output struct { + Capability *clients.Capability `json:"capability,omitempty" jsonschema:"The created or updated capability."` + Success bool `json:"success" jsonschema:"Whether the capability was saved successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if saving the capability failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "create_capability", + Title: "Create or Update Edge Capability", + Description: "Creates or updates an Edge capability (e.g. jdbc-ingestion) via the private Edge capability management API. Does not run the capability — use start_ingestion to trigger a jdbc-ingestion run.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUIDOptional("capabilityId", input.CapabilityID); err != nil { + return Output{}, err + } + if err := validation.UUID("edgeSiteId", input.EdgeSiteID); err != nil { + return Output{}, err + } + + parameters := input.Parameters + if parameters == nil { + parameters = map[string]any{} + } + + request := clients.CapabilityRequest{ + TypeID: input.TypeID, + Name: input.Name, + Description: input.Description, + EdgeSiteID: input.EdgeSiteID, + Parameters: parameters, + } + + var capability *clients.Capability + var err error + if input.CapabilityID != "" { + capability, err = clients.CreateOrUpdateCapability(ctx, collibraClient, input.CapabilityID, request) + } else { + capability, err = clients.CreateCapability(ctx, collibraClient, request) + } + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to save capability: %s", err.Error())}, nil + } + + return Output{Capability: capability, Success: true}, nil + } +} diff --git a/pkg/tools/create_capability/tool_test.go b/pkg/tools/create_capability/tool_test.go new file mode 100644 index 0000000..6b3fd15 --- /dev/null +++ b/pkg/tools/create_capability/tool_test.go @@ -0,0 +1,95 @@ +package create_capability_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/create_capability" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestCreateCapability_Create(t *testing.T) { + siteID, _ := uuid.NewUUID() + capID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/capabilities", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityRequest) (int, clients.Capability) { + if in.TypeID != "jdbc-ingestion" { + t.Fatalf("unexpected typeId: %s", in.TypeID) + } + return http.StatusCreated, clients.Capability{ + ID: capID.String(), + Name: in.Name, + EdgeSiteID: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "local-jdbc-ingestion", + TypeID: "jdbc-ingestion", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{"connection": "some-connection-id", "message-mode": "KAFKA"}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Capability.ID != capID.String() { + t.Fatalf("expected id %s, got %s", capID.String(), output.Capability.ID) + } +} + +func TestCreateCapability_UpdateWithKnownID(t *testing.T) { + siteID, _ := uuid.NewUUID() + capID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("PUT /edge/api/rest/v2/capabilities/"+capID.String(), testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityRequest) (int, clients.Capability) { + return http.StatusOK, clients.Capability{ + ID: capID.String(), + Name: in.Name, + EdgeSiteID: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + CapabilityID: capID.String(), + Name: "local-jdbc-ingestion", + TypeID: "jdbc-ingestion", + EdgeSiteID: siteID.String(), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestCreateCapability_InvalidCapabilityID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + CapabilityID: "not-a-uuid", + Name: "bad", + TypeID: "jdbc-ingestion", + EdgeSiteID: "also-not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid capabilityId") + } +} diff --git a/pkg/tools/create_community/tool.go b/pkg/tools/create_community/tool.go new file mode 100644 index 0000000..7a3c96b --- /dev/null +++ b/pkg/tools/create_community/tool.go @@ -0,0 +1,49 @@ +// Package create_community implements the create_community MCP tool: creates a DGC +// community, the top-level catalog scaffolding required before creating domains and +// registering databases via configure_database. +package create_community + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name" jsonschema:"The community name."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the community."` +} + +type Output struct { + Community *clients.CommunityDetails `json:"community,omitempty" jsonschema:"The created community."` + Success bool `json:"success" jsonschema:"Whether the community was created successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if creating the community failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "create_community", + Title: "Create Community", + Description: "Creates a DGC community. A community is required before creating a domain (create_domain) and registering a database (configure_database).", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + community, err := clients.CreateCommunity(ctx, collibraClient, clients.CreateCommunityRequest{ + Name: input.Name, + Description: input.Description, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to create community: %s", err.Error())}, nil + } + return Output{Community: community, Success: true}, nil + } +} diff --git a/pkg/tools/create_community/tool_test.go b/pkg/tools/create_community/tool_test.go new file mode 100644 index 0000000..2fed332 --- /dev/null +++ b/pkg/tools/create_community/tool_test.go @@ -0,0 +1,61 @@ +package create_community_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/create_community" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestCreateCommunity(t *testing.T) { + communityID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /rest/2.0/communities", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CreateCommunityRequest) (int, clients.CommunityDetails) { + if in.Name != "Local Development" { + t.Fatalf("unexpected name: %s", in.Name) + } + return http.StatusCreated, clients.CommunityDetails{ID: communityID.String(), Name: in.Name, Description: in.Description} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "Local Development", + Description: "Local e2e test environment", + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Community.ID != communityID.String() { + t.Fatalf("expected id %s, got %s", communityID.String(), output.Community.ID) + } +} + +func TestCreateCommunity_Failure(t *testing.T) { + handler := http.NewServeMux() + handler.HandleFunc("POST /rest/2.0/communities", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Name: "bad"}) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure") + } +} diff --git a/pkg/tools/create_connection/tool.go b/pkg/tools/create_connection/tool.go new file mode 100644 index 0000000..25250df --- /dev/null +++ b/pkg/tools/create_connection/tool.go @@ -0,0 +1,164 @@ +// Package create_connection implements the create_connection MCP tool: creates or +// updates an Edge connection (e.g. a JDBC connection backing the jdbc-ingestion +// capability) via the private Edge connection management API. +package create_connection + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + ConnectionID string `json:"connectionId,omitempty" jsonschema:"Optional. UUID of the connection to create or update. If provided, the connection is created with this exact id (or updated if it already exists). If omitted, a new connection is created and the server assigns an id."` + Name string `json:"name" jsonschema:"The connection name."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the connection."` + TypeID string `json:"typeId" jsonschema:"The id of the connection type (e.g. 'Generic' for a generic JDBC connection, or a vendor-specific type such as a Snowflake connection type). Determines which parameters are expected — use list_capability_types to inspect a type's manifest."` + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this connection will be valid. Use the list_edge_sites tool to discover available sites."` + VaultID string `json:"vaultId,omitempty" jsonschema:"Optional UUID of the vault to retrieve vault-backed parameters from."` + Parameters map[string]any `json:"parameters" jsonschema:"Fixed connection parameters as defined by the connection type's manifest (e.g. driver-class, connection-string for the 'Generic' JDBC connection type). If driverJarUrl is provided, the driver-jar parameter is set automatically and does not need to be included here. Do NOT put open-ended/vendor-specific properties (e.g. a Snowflake connection's Role, Warehouse, User, Database, or a private key file) here — use additionalProperties instead. If you don't already know a data source's driver class, connection string format, or required properties, call get_data_source_setup_guide first instead of guessing."` + + DriverJarURL string `json:"driverJarUrl,omitempty" jsonschema:"Optional. A URL to download a JDBC driver jar from (e.g. a Maven Central artifact URL). If provided, the jar is downloaded and uploaded to the edge site, and the resulting artifact URI is set as the 'driver-jar' connection parameter."` + DriverJarFilename string `json:"driverJarFilename,omitempty" jsonschema:"Required if driverJarUrl is provided. The filename to upload the driver jar as (e.g. 'postgresql-42.7.11.jar')."` + + AdditionalProperties []AdditionalProperty `json:"additionalProperties,omitempty" jsonschema:"Optional. Open-ended, connection-type-defined properties beyond the fixed manifest parameters — e.g. a Snowflake connection (via the 'Generic' JDBC connection type) uses this for Role, Warehouse, User, Database, and private_key_file. Confirmed shape: each entry is injected as {name, type, value, secret} into a single array-valued parameter (see additionalPropertiesKey). For a FILE-type property, first call upload_file and use its returned artifact URI as this entry's value with type='file'."` + AdditionalPropertiesKey string `json:"additionalPropertiesKey,omitempty" jsonschema:"Optional. The manifest parameter name that additionalProperties are grouped under. Defaults to 'connection-properties', which is correct for the 'Generic' JDBC connection type (confirmed live). Other connection types may use a different key (e.g. AWS/GCP connections use 'additional-parameters') — check the type's manifest via list_capability_types if unsure."` +} + +// AdditionalProperty is one entry of an open-ended, connection-type-defined property +// list (DGC's ConnectionManifest.ParameterType.USER_DEFINED). Confirmed against a real +// working Snowflake-via-Generic-JDBC connection: each entry is {name, type, value, +// secret}, and a FILE-type entry's value is the bare artifact URI (same shape as any +// other FILE parameter), not a wrapped object. +type AdditionalProperty struct { + Name string `json:"name" jsonschema:"The property name (e.g. 'Role', 'Warehouse', 'private_key_file')."` + Type string `json:"type,omitempty" jsonschema:"'string' (default) or 'file'. Use 'file' when value is an artifact URI returned by upload_file."` + Value string `json:"value" jsonschema:"The property value. For type='file', this is the artifact URI returned by upload_file, not raw file content."` + Secret bool `json:"secret,omitempty" jsonschema:"Optional. Whether this value should be treated as a secret (e.g. a private key file or password)."` +} + +type Output struct { + Connection *clients.Connection `json:"connection,omitempty" jsonschema:"The created or updated connection."` + Success bool `json:"success" jsonschema:"Whether the connection was saved successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if saving the connection failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "create_connection", + Title: "Create or Update Edge Connection", + Description: "Creates or updates an Edge connection (e.g. a JDBC connection for the jdbc-ingestion capability) via the private Edge connection management API.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUIDOptional("connectionId", input.ConnectionID); err != nil { + return Output{}, err + } + if err := validation.UUID("edgeSiteId", input.EdgeSiteID); err != nil { + return Output{}, err + } + if err := validation.UUIDOptional("vaultId", input.VaultID); err != nil { + return Output{}, err + } + + parameters := input.Parameters + if parameters == nil { + parameters = map[string]any{} + } + + if input.DriverJarURL != "" { + if input.DriverJarFilename == "" { + return Output{}, fmt.Errorf("driverJarFilename is required when driverJarUrl is provided") + } + uri, err := downloadAndUploadDriver(ctx, collibraClient, input.DriverJarURL, input.DriverJarFilename) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to prepare driver jar: %s", err.Error())}, nil + } + parameters["driver-jar"] = uri + } + + if len(input.AdditionalProperties) > 0 { + key := input.AdditionalPropertiesKey + if key == "" { + key = "connection-properties" + } + entries := make([]map[string]any, len(input.AdditionalProperties)) + for i, prop := range input.AdditionalProperties { + if prop.Name == "" { + return Output{}, fmt.Errorf("additionalProperties[%d]: name is required", i) + } + propType := prop.Type + if propType == "" { + propType = "string" + } + entries[i] = map[string]any{ + "name": prop.Name, + "type": propType, + "value": prop.Value, + "secret": prop.Secret, + } + } + parameters[key] = entries + } + + request := clients.ConnectionRequest{ + Name: input.Name, + Description: input.Description, + TypeID: input.TypeID, + EdgeSiteID: input.EdgeSiteID, + VaultID: input.VaultID, + Parameters: parameters, + } + + var connection *clients.Connection + var err error + if input.ConnectionID != "" { + connection, err = clients.CreateOrUpdateConnection(ctx, collibraClient, input.ConnectionID, request) + } else { + connection, err = clients.CreateConnection(ctx, collibraClient, request) + } + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to save connection: %s", err.Error())}, nil + } + + return Output{Connection: connection, Success: true}, nil + } +} + +// downloadAndUploadDriver fetches the driver jar bytes from an arbitrary external URL +// (not routed through the Collibra client, since it points outside the tenant) and +// re-uploads them to the edge site via the authenticated Collibra client. +func downloadAndUploadDriver(ctx context.Context, collibraClient *http.Client, url, filename string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", fmt.Errorf("building driver download request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", fmt.Errorf("downloading driver jar: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("downloading driver jar: unexpected status %d", resp.StatusCode) + } + + content, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("reading driver jar content: %w", err) + } + + return clients.UploadFile(ctx, collibraClient, filename, content) +} diff --git a/pkg/tools/create_connection/tool_test.go b/pkg/tools/create_connection/tool_test.go new file mode 100644 index 0000000..4aca41c --- /dev/null +++ b/pkg/tools/create_connection/tool_test.go @@ -0,0 +1,231 @@ +package create_connection_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/create_connection" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestCreateConnection_Create(t *testing.T) { + siteID, _ := uuid.NewUUID() + connID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + if in.Name != "local-postgres-source" { + t.Fatalf("unexpected name: %s", in.Name) + } + if in.EdgeSiteID != siteID.String() { + t.Fatalf("unexpected edgeSiteId: %s", in.EdgeSiteID) + } + return http.StatusCreated, clients.Connection{ + ID: connID.String(), + Name: in.Name, + TypeID: in.TypeID, + EdgeSiteID: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "local-postgres-source", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{"driver-class": "org.postgresql.Driver"}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Connection.ID != connID.String() { + t.Fatalf("expected id %s, got %s", connID.String(), output.Connection.ID) + } +} + +func TestCreateConnection_UpdateWithKnownID(t *testing.T) { + siteID, _ := uuid.NewUUID() + connID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("PUT /edge/api/rest/v2/connections/"+connID.String(), testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + return http.StatusOK, clients.Connection{ + ID: connID.String(), + Name: in.Name, + TypeID: in.TypeID, + EdgeSiteID: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + ConnectionID: connID.String(), + Name: "local-postgres-source", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Connection.ID != connID.String() { + t.Fatalf("expected id %s, got %s", connID.String(), output.Connection.ID) + } +} + +func TestCreateConnection_InvalidEdgeSiteID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "bad", + TypeID: "Generic", + EdgeSiteID: "not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid edgeSiteId") + } +} + +func TestCreateConnection_AdditionalProperties(t *testing.T) { + // Mirrors a real, confirmed-working Snowflake-via-Generic-JDBC connection's + // parameters: driver-class/connection-string/driver-jar fixed, plus a + // "connection-properties" array of {name, type, value, secret} for Role, + // Warehouse, private_key_file, User, Database. + siteID, _ := uuid.NewUUID() + connID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + props, ok := in.Parameters["connection-properties"].([]any) + if !ok || len(props) != 2 { + t.Fatalf("expected 2 connection-properties entries, got: %#v", in.Parameters["connection-properties"]) + } + role, ok := props[0].(map[string]any) + if !ok || role["name"] != "Role" || role["type"] != "string" || role["value"] != "SOFTWARE_DEVELOPMENT" || role["secret"] != false { + t.Fatalf("unexpected Role entry: %#v", props[0]) + } + keyFile, ok := props[1].(map[string]any) + if !ok || keyFile["name"] != "private_key_file" || keyFile["type"] != "file" || keyFile["value"] != "artifact://uuid/key.p8" || keyFile["secret"] != true { + t.Fatalf("unexpected private_key_file entry: %#v", props[1]) + } + return http.StatusCreated, clients.Connection{ID: connID.String(), Name: in.Name, EdgeSiteID: in.EdgeSiteID, Parameters: in.Parameters} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "snowflake-jdbc", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{ + "driver-class": "net.snowflake.client.jdbc.SnowflakeDriver", + "connection-string": "jdbc:snowflake://acme.snowflakecomputing.com", + }, + AdditionalProperties: []tools.AdditionalProperty{ + {Name: "Role", Value: "SOFTWARE_DEVELOPMENT"}, + {Name: "private_key_file", Type: "file", Value: "artifact://uuid/key.p8", Secret: true}, + }, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestCreateConnection_AdditionalProperties_CustomKey(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + if _, ok := in.Parameters["additional-parameters"]; !ok { + t.Fatalf("expected additional-parameters key, got parameters: %#v", in.Parameters) + } + return http.StatusCreated, clients.Connection{Parameters: in.Parameters} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "aws-conn", + TypeID: "AWS", + EdgeSiteID: siteID.String(), + AdditionalPropertiesKey: "additional-parameters", + AdditionalProperties: []tools.AdditionalProperty{{Name: "region", Value: "us-east-1"}}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestCreateConnection_DriverJarUpload(t *testing.T) { + siteID, _ := uuid.NewUUID() + connID, _ := uuid.NewUUID() + + var uploadedJarURI string + handler := http.NewServeMux() + handler.HandleFunc("POST /edge/api/rest/v2/upload", func(w http.ResponseWriter, r *http.Request) { + uploadedJarURI = "jar://test-uuid/driver.jar" + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`"` + uploadedJarURI + `"`)) + }) + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + if in.Parameters["driver-jar"] != uploadedJarURI { + t.Fatalf("expected driver-jar %q, got %v", uploadedJarURI, in.Parameters["driver-jar"]) + } + return http.StatusCreated, clients.Connection{ + ID: connID.String(), + Name: in.Name, + EdgeSiteID: in.EdgeSiteID, + Parameters: in.Parameters, + } + })) + + edgeServer := httptest.NewServer(handler) + defer edgeServer.Close() + + driverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("fake jar bytes")) + })) + defer driverServer.Close() + + client := testutil.NewClient(edgeServer) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "local-postgres-source", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + DriverJarURL: driverServer.URL, + DriverJarFilename: "driver.jar", + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} diff --git a/pkg/tools/create_domain/tool.go b/pkg/tools/create_domain/tool.go new file mode 100644 index 0000000..1018ac0 --- /dev/null +++ b/pkg/tools/create_domain/tool.go @@ -0,0 +1,63 @@ +// Package create_domain implements the create_domain MCP tool: creates a DGC domain +// within a community, e.g. a "Physical Data Dictionary" domain required before +// registering a database via configure_database. +package create_domain + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name" jsonschema:"The domain name."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the domain."` + CommunityID string `json:"communityId" jsonschema:"UUID of the community (created via create_community) this domain belongs to."` + TypeID string `json:"typeId" jsonschema:"UUID of the domain type. There is no discovery endpoint for this in DGC's public API; the well-known 'Physical Data Dictionary' domain type id (00000000-0000-0000-0000-000000030011), used for jdbc-ingestion databases, is stable across DGC versions."` + ExcludedFromAutoHyperlinking bool `json:"excludedFromAutoHyperlinking,omitempty" jsonschema:"Optional. If true, assets in this domain are excluded from automatic hyperlinking."` +} + +type Output struct { + Domain *clients.DomainDetails `json:"domain,omitempty" jsonschema:"The created domain."` + Success bool `json:"success" jsonschema:"Whether the domain was created successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if creating the domain failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "create_domain", + Title: "Create Domain", + Description: "Creates a DGC domain within a community, e.g. a Physical Data Dictionary domain required before registering a database via configure_database.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("communityId", input.CommunityID); err != nil { + return Output{}, err + } + if err := validation.UUID("typeId", input.TypeID); err != nil { + return Output{}, err + } + + domain, err := clients.CreateDomain(ctx, collibraClient, clients.CreateDomainRequest{ + Name: input.Name, + Description: input.Description, + CommunityID: input.CommunityID, + TypeID: input.TypeID, + ExcludedFromAutoHyperlinking: input.ExcludedFromAutoHyperlinking, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to create domain: %s", err.Error())}, nil + } + return Output{Domain: domain, Success: true}, nil + } +} diff --git a/pkg/tools/create_domain/tool_test.go b/pkg/tools/create_domain/tool_test.go new file mode 100644 index 0000000..b54590d --- /dev/null +++ b/pkg/tools/create_domain/tool_test.go @@ -0,0 +1,62 @@ +package create_domain_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/create_domain" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestCreateDomain(t *testing.T) { + communityID, _ := uuid.NewUUID() + typeID, _ := uuid.NewUUID() + domainID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /rest/2.0/domains", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CreateDomainRequest) (int, clients.DomainDetails) { + if in.CommunityID != communityID.String() { + t.Fatalf("unexpected communityId: %s", in.CommunityID) + } + return http.StatusCreated, clients.DomainDetails{ + ID: domainID.String(), + Name: in.Name, + CommunityID: in.CommunityID, + TypeID: in.TypeID, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "postgres-src [Physical Data Dictionary]", + CommunityID: communityID.String(), + TypeID: typeID.String(), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Domain.ID != domainID.String() { + t.Fatalf("expected id %s, got %s", domainID.String(), output.Domain.ID) + } +} + +func TestCreateDomain_InvalidCommunityID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Name: "bad", + CommunityID: "not-a-uuid", + TypeID: "also-not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid communityId") + } +} diff --git a/pkg/tools/get_data_source_setup_guide/tool.go b/pkg/tools/get_data_source_setup_guide/tool.go new file mode 100644 index 0000000..15923ac --- /dev/null +++ b/pkg/tools/get_data_source_setup_guide/tool.go @@ -0,0 +1,124 @@ +// Package get_data_source_setup_guide implements the get_data_source_setup_guide MCP +// tool: looks up driver class, JDBC connection string format, and per-auth-method +// connection properties for any of Collibra's ~68 supported data sources, so +// create_connection can be filled in correctly without the agent (or the user) +// needing to already know a given vendor's JDBC driver properties. +// +// This is a documented gap, not a guaranteed integration: the underlying data comes +// from scraping Collibra's public product documentation site (see the package doc +// on clients.GetConnectionProperties for the full explanation and how to re-derive +// it if it breaks). There is no published API for this — Collibra could restructure +// their docs site at any time without notice. Treat a failure from this tool as "ask +// the user for the connection properties directly," not a bug to retry around. +package get_data_source_setup_guide + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + DataSource string `json:"dataSource" jsonschema:"The data source to look up (e.g. 'Snowflake', 'PostgreSQL', 'Amazon Redshift'). Matched case-insensitively against Collibra's ~68 supported data sources; if ambiguous or not found, the response lists candidates to retry with."` +} + +type Output struct { + DataSourceID string `json:"dataSourceId,omitempty" jsonschema:"The matched data source's internal id (slug)."` + DataSourceName string `json:"dataSourceName,omitempty" jsonschema:"The matched data source's display name."` + DriverClassName string `json:"driverClassName,omitempty" jsonschema:"The JDBC driver class name for create_connection's 'driver-class' parameter."` + ConnectionStringFormat string `json:"connectionStringFormat,omitempty" jsonschema:"The JDBC connection string format for create_connection's 'connection-string' parameter, with placeholders (e.g. 'jdbc:snowflake://.snowflakecomputing.com')."` + MarketplaceURL string `json:"marketplaceUrl,omitempty" jsonschema:"Collibra Marketplace listing URL for this data source's JDBC driver, if the driver needs to be downloaded before uploading via create_connection's driverJarUrl/driverJarFilename or the upload_file tool."` + Prerequisites string `json:"prerequisites,omitempty" jsonschema:"Documented prerequisites before creating this connection."` + AuthMethods []clients.AuthMethodProperties `json:"authMethods,omitempty" jsonschema:"Documented connection properties, grouped by supported authentication method. Pick the method matching what credentials the user has, then pass its properties as create_connection's additionalProperties (name/type/value/secret per entry — secret=true when the property's Value Type here is 'Secret'). A 'File' type property's value must first be uploaded via upload_file or create_connection's driverJarUrl, then referenced by the returned artifact URI."` + Matches []clients.DataSourceSlug `json:"matches,omitempty" jsonschema:"Candidate data sources when the lookup was ambiguous or not found. Retry with one of these exact ids."` + Success bool `json:"success" jsonschema:"Whether at least some setup information was found."` + Error string `json:"error,omitempty" jsonschema:"Details if part or all of the lookup failed. A failure here does not mean the data source is unsupported by Edge — it means this best-effort documentation lookup didn't find it; ask the user for the connection properties directly."` +} + +func NewTool(_ *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_data_source_setup_guide", + Title: "Get Data Source Connection Setup Guide", + Description: "Looks up driver class, JDBC connection string format, and per-auth-method connection properties for a data source, from Collibra's public documentation (best-effort scrape, not a stable API — see tool source for details). Use before create_connection when the required parameters for a data source (e.g. Snowflake's Role/Warehouse/private key) aren't already known.", + Handler: handler(), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +// DocsClient hits Collibra's public documentation site directly, never the tenant's +// Collibra instance — it must not use the injected, tenant-scoped Collibra client. +// Overridable in tests. +var DocsClient = http.DefaultClient + +func handler() chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if strings.TrimSpace(input.DataSource) == "" { + return Output{}, fmt.Errorf("dataSource is required") + } + + sources, err := clients.ListDataSources(ctx, DocsClient) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to list data sources: %s", err.Error())}, nil + } + + matches := matchDataSources(sources, input.DataSource) + if len(matches) == 0 { + return Output{Success: false, Error: fmt.Sprintf("no data source found matching %q", input.DataSource), Matches: sources}, nil + } + if len(matches) > 1 { + return Output{Success: false, Error: fmt.Sprintf("multiple data sources matched %q; retry with one exact id", input.DataSource), Matches: matches}, nil + } + + slug := matches[0] + output := Output{DataSourceID: slug.ID, DataSourceName: slug.Name} + + var errs []string + + if info, err := clients.GetDataSourceInfo(ctx, DocsClient, slug.ID); err != nil { + errs = append(errs, fmt.Sprintf("driver/connection-string info unavailable: %s", err.Error())) + } else { + output.DriverClassName = info.DriverClassName + output.ConnectionStringFormat = info.ConnectionStringFormat + output.MarketplaceURL = info.MarketplaceURL + output.Prerequisites = info.Prerequisites + } + + if methods, err := clients.GetConnectionProperties(ctx, DocsClient, slug.ID); err != nil { + errs = append(errs, fmt.Sprintf("connection properties unavailable: %s", err.Error())) + } else { + output.AuthMethods = methods + } + + output.Success = output.DriverClassName != "" || len(output.AuthMethods) > 0 + output.Error = strings.Join(errs, "; ") + return output, nil + } +} + +// matchDataSources resolves a free-text query against the known data source list: +// exact id/name match wins outright; otherwise falls back to substring matches. +func matchDataSources(sources []clients.DataSourceSlug, query string) []clients.DataSourceSlug { + q := strings.ToLower(strings.TrimSpace(query)) + + var exact, partial []clients.DataSourceSlug + for _, s := range sources { + id, name := strings.ToLower(s.ID), strings.ToLower(s.Name) + switch { + case id == q || name == q: + exact = append(exact, s) + case strings.Contains(id, q) || strings.Contains(name, q): + partial = append(partial, s) + } + } + + if len(exact) > 0 { + return exact + } + return partial +} diff --git a/pkg/tools/get_data_source_setup_guide/tool_test.go b/pkg/tools/get_data_source_setup_guide/tool_test.go new file mode 100644 index 0000000..0362e3f --- /dev/null +++ b/pkg/tools/get_data_source_setup_guide/tool_test.go @@ -0,0 +1,139 @@ +package get_data_source_setup_guide_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/get_data_source_setup_guide" + "github.com/collibra/chip/pkg/tools/testutil" +) + +const selectorFixture = ` +
      +
    • Snowflake
      • +
    • PostgreSQL
      • +
    • PostgreSQL Classic
      • +
    +` + +const dataSourceInfoFixture = ` +

    The JDBC connection string:

    +

    jdbc:snowflake://<accountname>.snowflakecomputing.com

    +

    The JDBC driver class name:

    +
    net.snowflake.client.jdbc.SnowflakeDriver
    +` + +const connectionPropertiesFixture = ` +
    + +

    Required?

    Name

    Type

    Value Type

    Value

    YesPasswordTextSecretThe password.
    +
    +` + +func newTestServer(t *testing.T) *http.Client { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/_all-data-sources/selector/selector.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(selectorFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/snowflake/data-source-information.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(dataSourceInfoFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/snowflake/minimum-connection-properties-edge.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(connectionPropertiesFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/postgresql/data-source-information.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(dataSourceInfoFixture)) + }) + mux.HandleFunc("GET /docs/catalog-connectors/Content/Resources/Snippets/CatalogSnippets/CatalogConnectors/postgresql/minimum-connection-properties-edge.htm", func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(connectionPropertiesFixture)) + }) + return testutil.NewClient(httptest.NewServer(mux)) +} + +func TestGetDataSourceSetupGuide_Success(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "Snowflake"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.DataSourceID != "snowflake" { + t.Fatalf("unexpected data source id: %s", output.DataSourceID) + } + if output.DriverClassName != "net.snowflake.client.jdbc.SnowflakeDriver" { + t.Fatalf("unexpected driver class: %s", output.DriverClassName) + } + if len(output.AuthMethods) != 1 || len(output.AuthMethods[0].Properties) != 1 { + t.Fatalf("unexpected auth methods: %+v", output.AuthMethods) + } +} + +func TestGetDataSourceSetupGuide_CaseInsensitiveMatch(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "snowflake"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestGetDataSourceSetupGuide_Ambiguous(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "postgresql"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + // "postgresql" exact-matches one id but is also a substring of "postgresql-classic"; + // exact match should win outright rather than reporting ambiguity. + if !output.Success || output.DataSourceID != "postgresql" { + t.Fatalf("expected exact id match to win, got: %+v", output) + } +} + +func TestGetDataSourceSetupGuide_TrulyAmbiguous(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "postgres"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Success { + t.Fatalf("expected failure due to ambiguity") + } + if len(output.Matches) != 2 { + t.Fatalf("expected 2 candidate matches, got: %+v", output.Matches) + } +} + +func TestGetDataSourceSetupGuide_NotFound(t *testing.T) { + tools.DocsClient = newTestServer(t) + + output, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: "sap-hana"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Success { + t.Fatalf("expected failure for unknown data source") + } + if len(output.Matches) != 3 { + t.Fatalf("expected all 3 known sources returned as candidates, got: %+v", output.Matches) + } +} + +func TestGetDataSourceSetupGuide_EmptyInput(t *testing.T) { + tools.DocsClient = newTestServer(t) + + _, err := tools.NewTool(nil).Handler(t.Context(), tools.Input{DataSource: ""}) + if err == nil { + t.Fatalf("expected an error for empty dataSource") + } +} diff --git a/pkg/tools/get_job_status/tool.go b/pkg/tools/get_job_status/tool.go new file mode 100644 index 0000000..2f7f4f2 --- /dev/null +++ b/pkg/tools/get_job_status/tool.go @@ -0,0 +1,49 @@ +// Package get_job_status implements the get_job_status MCP tool: polls the status of +// an Edge job, e.g. one started by test_connection or start_ingestion. +package get_job_status + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + JobID string `json:"jobId" jsonschema:"UUID of the job to check, as returned by test_connection or start_ingestion."` +} + +type Output struct { + Status string `json:"status,omitempty" jsonschema:"The job's current status: UNKNOWN, SUBMITTED, DOWNLOADING, SCHEDULED, RUNNING, SUCCEEDED, FAILED, CANCELLED, CANCELLING, CAPABILITY_PROGRESS, CAPABILITY_SUCCEEDED, or CAPABILITY_FAILED. SUCCEEDED/FAILED/CANCELLED/CAPABILITY_SUCCEEDED/CAPABILITY_FAILED are terminal."` + Message string `json:"message,omitempty" jsonschema:"Status message from the job."` + LastUpdatedDateTime string `json:"lastUpdatedDateTime,omitempty" jsonschema:"When the status was last updated."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_job_status", + Title: "Get Edge Job Status", + Description: "Gets the current status of an Edge job, e.g. one started by test_connection or start_ingestion.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("jobId", input.JobID); err != nil { + return Output{}, err + } + + status, err := clients.GetJobStatusLog(ctx, collibraClient, input.JobID) + if err != nil { + return Output{}, err + } + + return Output{Status: status.Status, Message: status.Message, LastUpdatedDateTime: status.LastUpdatedDateTime}, nil + } +} diff --git a/pkg/tools/get_job_status/tool_test.go b/pkg/tools/get_job_status/tool_test.go new file mode 100644 index 0000000..9d77209 --- /dev/null +++ b/pkg/tools/get_job_status/tool_test.go @@ -0,0 +1,46 @@ +package get_job_status_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/get_job_status" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestGetJobStatus(t *testing.T) { + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/jobs/"+jobID.String()+"/statusLog", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobStatusLog) { + return http.StatusOK, clients.JobStatusLog{ + JobID: jobID.String(), + Status: "CAPABILITY_SUCCEEDED", + Message: "done", + LastUpdatedDateTime: "2026-07-04T10:00:00.000Z", + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: jobID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Status != "CAPABILITY_SUCCEEDED" { + t.Fatalf("unexpected status: %s", output.Status) + } +} + +func TestGetJobStatus_InvalidJobID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid jobId") + } +} diff --git a/pkg/tools/list_capability_types/tool.go b/pkg/tools/list_capability_types/tool.go new file mode 100644 index 0000000..985f86f --- /dev/null +++ b/pkg/tools/list_capability_types/tool.go @@ -0,0 +1,54 @@ +// Package list_capability_types implements the list_capability_types MCP tool: a +// read-only discovery tool for finding valid typeId values (and their manifests) for +// create_capability and create_connection on a given edge site. +package list_capability_types + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site to list available capability and connection types for. Use the list_edge_sites tool to discover available sites."` +} + +type Output struct { + CapabilityTypes []clients.CapabilityType `json:"capabilityTypes" jsonschema:"Capability types available on this edge site (e.g. 'jdbc-ingestion'), with their manifests describing expected install parameters."` + ConnectionTypes []clients.ConnectionType `json:"connectionTypes" jsonschema:"Connection types available on this edge site (e.g. 'Generic' or a vendor-specific type), with their manifests describing expected parameters."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "list_capability_types", + Title: "List Capability and Connection Types", + Description: "Lists the capability and connection types available on an edge site, including each type's manifest. Use this to find the typeId and expected parameters for create_capability and create_connection.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("edgeSiteId", input.EdgeSiteID); err != nil { + return Output{}, err + } + + capabilityTypes, err := clients.GetCapabilityTypes(ctx, collibraClient, input.EdgeSiteID) + if err != nil { + return Output{}, err + } + + connectionTypes, err := clients.GetConnectionTypes(ctx, collibraClient, input.EdgeSiteID) + if err != nil { + return Output{}, err + } + + return Output{CapabilityTypes: capabilityTypes, ConnectionTypes: connectionTypes}, nil + } +} diff --git a/pkg/tools/list_capability_types/tool_test.go b/pkg/tools/list_capability_types/tool_test.go new file mode 100644 index 0000000..18dc771 --- /dev/null +++ b/pkg/tools/list_capability_types/tool_test.go @@ -0,0 +1,47 @@ +package list_capability_types_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/list_capability_types" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestListCapabilityTypes(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/capabilityTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.CapabilityType) { + return http.StatusOK, []clients.CapabilityType{{ID: "jdbc-ingestion"}} + })) + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/connectionTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.ConnectionType) { + return http.StatusOK, []clients.ConnectionType{{ID: "Generic"}} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: siteID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.CapabilityTypes) != 1 || output.CapabilityTypes[0].ID != "jdbc-ingestion" { + t.Fatalf("unexpected capability types: %+v", output.CapabilityTypes) + } + if len(output.ConnectionTypes) != 1 || output.ConnectionTypes[0].ID != "Generic" { + t.Fatalf("unexpected connection types: %+v", output.ConnectionTypes) + } +} + +func TestListCapabilityTypes_InvalidEdgeSiteID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid edgeSiteId") + } +} diff --git a/pkg/tools/list_edge_sites/tool.go b/pkg/tools/list_edge_sites/tool.go new file mode 100644 index 0000000..e9c87ea --- /dev/null +++ b/pkg/tools/list_edge_sites/tool.go @@ -0,0 +1,40 @@ +// Package list_edge_sites implements the list_edge_sites MCP tool: a read-only +// discovery tool for finding the edgeSiteId needed by create_connection and +// create_capability. +package list_edge_sites + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct{} + +type Output struct { + Sites []clients.EdgeSite `json:"sites" jsonschema:"The list of available edge sites."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "list_edge_sites", + Title: "List Edge Sites", + Description: "Lists available Edge sites, including their id, name, and status. Use this to find the edgeSiteId needed by create_connection and create_capability.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, _ Input) (Output, error) { + sites, err := clients.GetEdgeSites(ctx, collibraClient) + if err != nil { + return Output{}, err + } + return Output{Sites: sites}, nil + } +} diff --git a/pkg/tools/list_edge_sites/tool_test.go b/pkg/tools/list_edge_sites/tool_test.go new file mode 100644 index 0000000..93a1d39 --- /dev/null +++ b/pkg/tools/list_edge_sites/tool_test.go @@ -0,0 +1,38 @@ +package list_edge_sites_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/list_edge_sites" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestListEdgeSites(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/sites", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeSite) { + return http.StatusOK, []clients.EdgeSite{ + {ID: siteID.String(), Name: "local-dev", Status: "HEALTHY"}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.Sites) != 1 { + t.Fatalf("expected 1 site, got: %d", len(output.Sites)) + } + if output.Sites[0].Name != "local-dev" { + t.Fatalf("expected name 'local-dev', got: '%s'", output.Sites[0].Name) + } +} diff --git a/pkg/tools/register.go b/pkg/tools/register.go index fee28b6..0154e4d 100644 --- a/pkg/tools/register.go +++ b/pkg/tools/register.go @@ -7,7 +7,12 @@ import ( "github.com/collibra/chip/pkg/chip" "github.com/collibra/chip/pkg/skills" "github.com/collibra/chip/pkg/tools/add_data_classification_match" + "github.com/collibra/chip/pkg/tools/configure_database" "github.com/collibra/chip/pkg/tools/create_asset" + "github.com/collibra/chip/pkg/tools/create_capability" + "github.com/collibra/chip/pkg/tools/create_community" + "github.com/collibra/chip/pkg/tools/create_connection" + "github.com/collibra/chip/pkg/tools/create_domain" "github.com/collibra/chip/pkg/tools/discover_business_glossary" "github.com/collibra/chip/pkg/tools/discover_data_assets" "github.com/collibra/chip/pkg/tools/edit_asset" @@ -15,7 +20,9 @@ import ( "github.com/collibra/chip/pkg/tools/get_business_term_data" "github.com/collibra/chip/pkg/tools/get_column_semantics" "github.com/collibra/chip/pkg/tools/get_context_specification" + "github.com/collibra/chip/pkg/tools/get_data_source_setup_guide" "github.com/collibra/chip/pkg/tools/get_debug_mcp_init_request" + "github.com/collibra/chip/pkg/tools/get_job_status" "github.com/collibra/chip/pkg/tools/get_lineage_downstream" "github.com/collibra/chip/pkg/tools/get_lineage_entity" "github.com/collibra/chip/pkg/tools/get_lineage_transformation" @@ -24,8 +31,10 @@ import ( "github.com/collibra/chip/pkg/tools/get_table_semantics" "github.com/collibra/chip/pkg/tools/init_data_contract" "github.com/collibra/chip/pkg/tools/list_asset_types" + "github.com/collibra/chip/pkg/tools/list_capability_types" "github.com/collibra/chip/pkg/tools/list_context_specifications" "github.com/collibra/chip/pkg/tools/list_data_contracts" + "github.com/collibra/chip/pkg/tools/list_edge_sites" "github.com/collibra/chip/pkg/tools/prepare_create_asset" "github.com/collibra/chip/pkg/tools/pull_data_contract_manifest" "github.com/collibra/chip/pkg/tools/push_data_contract_manifest" @@ -35,6 +44,9 @@ import ( "github.com/collibra/chip/pkg/tools/search_data_classification_matches" "github.com/collibra/chip/pkg/tools/search_lineage_entities" "github.com/collibra/chip/pkg/tools/search_lineage_transformations" + "github.com/collibra/chip/pkg/tools/start_ingestion" + "github.com/collibra/chip/pkg/tools/test_connection" + "github.com/collibra/chip/pkg/tools/upload_file" ) // ContextSpecificationsFeature is the experimental-feature identifier used to @@ -80,6 +92,18 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, list_context_specifications.NewTool(client)) toolRegister(server, toolConfig, get_context_specification.NewTool(client)) } + toolRegister(server, toolConfig, list_edge_sites.NewTool(client)) + toolRegister(server, toolConfig, list_capability_types.NewTool(client)) + toolRegister(server, toolConfig, create_community.NewTool(client)) + toolRegister(server, toolConfig, create_domain.NewTool(client)) + toolRegister(server, toolConfig, upload_file.NewTool(client)) + toolRegister(server, toolConfig, get_data_source_setup_guide.NewTool(client)) + toolRegister(server, toolConfig, create_connection.NewTool(client)) + toolRegister(server, toolConfig, create_capability.NewTool(client)) + toolRegister(server, toolConfig, test_connection.NewTool(client)) + toolRegister(server, toolConfig, get_job_status.NewTool(client)) + toolRegister(server, toolConfig, configure_database.NewTool(client)) + toolRegister(server, toolConfig, start_ingestion.NewTool(client)) if toolConfig.EnableDebugTools { toolRegister(server, toolConfig, get_debug_mcp_init_request.NewTool(client)) diff --git a/pkg/tools/start_ingestion/tool.go b/pkg/tools/start_ingestion/tool.go new file mode 100644 index 0000000..2156cc0 --- /dev/null +++ b/pkg/tools/start_ingestion/tool.go @@ -0,0 +1,54 @@ +// Package start_ingestion implements the start_ingestion MCP tool: triggers the +// jdbc-ingestion capability run for a registered Database asset. +package start_ingestion + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + DatabaseID string `json:"databaseId" jsonschema:"UUID of the Database asset to synchronize (as registered by configure_database)."` + SchemaConnectionIDs []string `json:"schemaConnectionIds,omitempty" jsonschema:"Optional. UUIDs of specific schema connections to synchronize. If omitted, all schemas that already have synchronization rules configured (via configure_database) are synchronized."` +} + +type Output struct { + Job *clients.Job `json:"job,omitempty" jsonschema:"The started synchronization job."` + Success bool `json:"success" jsonschema:"Whether the ingestion job was started."` + Error string `json:"error,omitempty" jsonschema:"Error message if starting the job failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "start_ingestion", + Title: "Start Database Ingestion", + Description: "Triggers the jdbc-ingestion capability run for a registered Database asset, synchronizing its configured schemas/tables into the catalog. The database must already be registered and configured via configure_database.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("databaseId", input.DatabaseID); err != nil { + return Output{}, err + } + if err := validation.UUIDs("schemaConnectionIds", input.SchemaConnectionIDs); err != nil { + return Output{}, err + } + + job, err := clients.SynchronizeDatabaseMetadata(ctx, collibraClient, input.DatabaseID, input.SchemaConnectionIDs) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to start ingestion: %s", err.Error())}, nil + } + + return Output{Job: job, Success: true}, nil + } +} diff --git a/pkg/tools/start_ingestion/tool_test.go b/pkg/tools/start_ingestion/tool_test.go new file mode 100644 index 0000000..41467c9 --- /dev/null +++ b/pkg/tools/start_ingestion/tool_test.go @@ -0,0 +1,103 @@ +package start_ingestion_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/start_ingestion" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestStartIngestion_Success(t *testing.T) { + databaseID, _ := uuid.NewUUID() + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /rest/catalogDatabase/v1/databases/"+databaseID.String()+"/synchronizeMetadata", + testutil.JsonHandlerInOut(func(r *http.Request, in clients.DatabaseMetadataSynchronizationRequest) (int, clients.Job) { + if len(in.SchemaConnectionIDs) != 0 { + t.Fatalf("expected no schemaConnectionIds, got: %v", in.SchemaConnectionIDs) + } + return http.StatusAccepted, clients.Job{ID: jobID.String(), State: "RUNNING"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseID: databaseID.String(), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Job.ID != jobID.String() { + t.Fatalf("expected job id %s, got %s", jobID.String(), output.Job.ID) + } +} + +func TestStartIngestion_WithSchemaConnectionIDs(t *testing.T) { + databaseID, _ := uuid.NewUUID() + schemaConnID, _ := uuid.NewUUID() + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /rest/catalogDatabase/v1/databases/"+databaseID.String()+"/synchronizeMetadata", + testutil.JsonHandlerInOut(func(r *http.Request, in clients.DatabaseMetadataSynchronizationRequest) (int, clients.Job) { + if len(in.SchemaConnectionIDs) != 1 || in.SchemaConnectionIDs[0] != schemaConnID.String() { + t.Fatalf("unexpected schemaConnectionIds: %v", in.SchemaConnectionIDs) + } + return http.StatusAccepted, clients.Job{ID: jobID.String(), State: "RUNNING"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + DatabaseID: databaseID.String(), + SchemaConnectionIDs: []string{schemaConnID.String()}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestStartIngestion_InvalidDatabaseID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{DatabaseID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid databaseId") + } +} + +func TestStartIngestion_Conflict(t *testing.T) { + databaseID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("POST /rest/catalogDatabase/v1/databases/"+databaseID.String()+"/synchronizeMetadata", + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusConflict) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{DatabaseID: databaseID.String()}) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure on conflict") + } +} diff --git a/pkg/tools/test_connection/tool.go b/pkg/tools/test_connection/tool.go new file mode 100644 index 0000000..49cf5ee --- /dev/null +++ b/pkg/tools/test_connection/tool.go @@ -0,0 +1,57 @@ +// Package test_connection implements the test_connection MCP tool: verifies an Edge +// connection can actually reach its data source. +package test_connection + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + ConnectionID string `json:"connectionId" jsonschema:"UUID of the connection to test (created via create_connection)."` + TimeoutSec int `json:"timeoutSec,omitempty" jsonschema:"Optional. If provided and greater than 0, waits synchronously for the test to finish (up to this many seconds) and returns the final result. If omitted, returns immediately with a jobId while the test runs in the background — poll it with get_job_status."` +} + +type Output struct { + JobID string `json:"jobId,omitempty" jsonschema:"The id of the connection-test job. Use with get_job_status to poll for the result if timeoutSec was not provided."` + Success bool `json:"success" jsonschema:"Whether the test connection request itself succeeded. If timeoutSec was omitted, this reflects that the test job was submitted, not the connection test's outcome — check get_job_status for that."` + Message string `json:"message,omitempty" jsonschema:"Status message from the test."` + Error string `json:"error,omitempty" jsonschema:"Error message if the test request itself failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "test_connection", + Title: "Test Edge Connection", + Description: "Tests whether an Edge connection can reach its data source. Without timeoutSec, returns immediately with a jobId to poll via get_job_status; with timeoutSec, waits up to that many seconds and returns the final result.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("connectionId", input.ConnectionID); err != nil { + return Output{}, err + } + + var timeoutSec *int + if input.TimeoutSec > 0 { + timeoutSec = &input.TimeoutSec + } + + result, err := clients.TestConnection(ctx, collibraClient, input.ConnectionID, timeoutSec) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to test connection: %s", err.Error())}, nil + } + + return Output{JobID: result.JobID, Success: result.Success, Message: result.Message}, nil + } +} diff --git a/pkg/tools/test_connection/tool_test.go b/pkg/tools/test_connection/tool_test.go new file mode 100644 index 0000000..a25f386 --- /dev/null +++ b/pkg/tools/test_connection/tool_test.go @@ -0,0 +1,73 @@ +package test_connection_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/test_connection" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestTestConnection_Async(t *testing.T) { + connID, _ := uuid.NewUUID() + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/connections/"+connID.String()+"/test", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.TestConnectionResponse) { + if r.URL.Query().Get("timeoutSec") != "" { + t.Fatalf("expected no timeoutSec query param, got: %s", r.URL.Query().Get("timeoutSec")) + } + return http.StatusOK, clients.TestConnectionResponse{JobID: jobID.String(), Success: true, Message: "Job submitted successfully."} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ConnectionID: connID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success || output.JobID != jobID.String() { + t.Fatalf("unexpected output: %+v", output) + } +} + +func TestTestConnection_Synchronous(t *testing.T) { + connID, _ := uuid.NewUUID() + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/connections/"+connID.String()+"/test", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.TestConnectionResponse) { + if r.URL.Query().Get("timeoutSec") != "30" { + t.Fatalf("expected timeoutSec=30, got: %s", r.URL.Query().Get("timeoutSec")) + } + return http.StatusOK, clients.TestConnectionResponse{JobID: jobID.String(), Success: false, Message: "connection refused"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ConnectionID: connID.String(), TimeoutSec: 30}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Success { + t.Fatalf("expected the connection test itself to report failure") + } + if output.Message != "connection refused" { + t.Fatalf("unexpected message: %s", output.Message) + } +} + +func TestTestConnection_InvalidConnectionID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ConnectionID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid connectionId") + } +} diff --git a/pkg/tools/upload_file/tool.go b/pkg/tools/upload_file/tool.go new file mode 100644 index 0000000..5ed0971 --- /dev/null +++ b/pkg/tools/upload_file/tool.go @@ -0,0 +1,95 @@ +// Package upload_file implements the upload_file MCP tool: uploads an arbitrary file +// to an edge site and returns an artifact URI usable as the value of any FILE-type +// connection or capability parameter (JDBC drivers, TLS certs, private keys, keytabs, +// etc.) — not just the JDBC-driver convenience built into create_connection. +package upload_file + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Filename string `json:"filename" jsonschema:"The filename to upload as (e.g. 'client-cert.pem'). Determines the file extension DGC uses to infer the artifact type."` + URL string `json:"url,omitempty" jsonschema:"A URL to download the file's content from (e.g. a Maven Central artifact URL or an internal file server). Exactly one of url or contentBase64 must be provided."` + ContentBase64 string `json:"contentBase64,omitempty" jsonschema:"The file's content, base64-encoded. Exactly one of url or contentBase64 must be provided."` +} + +type Output struct { + URI string `json:"uri,omitempty" jsonschema:"The uploaded artifact URI (e.g. 'jar:///'). Use this as the value of a FILE-type connection or capability parameter."` + Success bool `json:"success" jsonschema:"Whether the file was uploaded successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if the upload failed."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "upload_file", + Title: "Upload File to Edge Site", + Description: "Uploads a file (JDBC driver, TLS certificate, private key, keytab, etc.) to an edge site and returns an artifact URI usable as the value of any FILE-type connection or capability parameter.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if input.Filename == "" { + return Output{}, fmt.Errorf("filename is required") + } + if (input.URL == "") == (input.ContentBase64 == "") { + return Output{}, fmt.Errorf("exactly one of url or contentBase64 must be provided") + } + + content, err := resolveContent(ctx, input) + if err != nil { + return Output{Success: false, Error: err.Error()}, nil + } + + uri, err := clients.UploadFile(ctx, collibraClient, input.Filename, content) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to upload file: %s", err.Error())}, nil + } + + return Output{URI: uri, Success: true}, nil + } +} + +func resolveContent(ctx context.Context, input Input) ([]byte, error) { + if input.ContentBase64 != "" { + content, err := base64.StdEncoding.DecodeString(input.ContentBase64) + if err != nil { + return nil, fmt.Errorf("decoding contentBase64: %w", err) + } + return content, nil + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, input.URL, nil) + if err != nil { + return nil, fmt.Errorf("building download request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("downloading file: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("downloading file: unexpected status %d", resp.StatusCode) + } + + content, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading downloaded content: %w", err) + } + + return content, nil +} diff --git a/pkg/tools/upload_file/tool_test.go b/pkg/tools/upload_file/tool_test.go new file mode 100644 index 0000000..c127f00 --- /dev/null +++ b/pkg/tools/upload_file/tool_test.go @@ -0,0 +1,84 @@ +package upload_file_test + +import ( + "encoding/base64" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/tools/testutil" + tools "github.com/collibra/chip/pkg/tools/upload_file" +) + +func TestUploadFile_FromBase64Content(t *testing.T) { + handler := http.NewServeMux() + handler.HandleFunc("POST /edge/api/rest/v2/upload", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("cert://test-uuid/client-cert.pem")) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Filename: "client-cert.pem", + ContentBase64: base64.StdEncoding.EncodeToString([]byte("fake cert bytes")), + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.URI != "cert://test-uuid/client-cert.pem" { + t.Fatalf("unexpected uri: %s", output.URI) + } +} + +func TestUploadFile_FromURL(t *testing.T) { + edgeHandler := http.NewServeMux() + edgeHandler.HandleFunc("POST /edge/api/rest/v2/upload", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("jar://test-uuid/driver.jar")) + }) + edgeServer := httptest.NewServer(edgeHandler) + defer edgeServer.Close() + + sourceServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("fake jar bytes")) + })) + defer sourceServer.Close() + + client := testutil.NewClient(edgeServer) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + Filename: "driver.jar", + URL: sourceServer.URL, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } +} + +func TestUploadFile_RequiresExactlyOneSource(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Filename: "x"}) + if err == nil { + t.Fatalf("expected an error when neither url nor contentBase64 is provided") + } + + _, err = tools.NewTool(client).Handler(t.Context(), tools.Input{ + Filename: "x", + URL: "http://example.com/x", + ContentBase64: "Zm9v", + }) + if err == nil { + t.Fatalf("expected an error when both url and contentBase64 are provided") + } +} From 743950e75b06b577c0b2dd7fa05cb8a196b46377 Mon Sep 17 00:00:00 2001 From: Dary Schaeken Date: Mon, 6 Jul 2026 13:47:56 +0200 Subject: [PATCH 02/10] feat: add discovery/lookup tools and fix rough edges found in live testing Adds five new tools surfaced as gaps during live end-to-end testing: find_users, find_domain_types, and find_connections (name-based lookups for ownerIds, create_domain's typeId, and picking up a manually-created connection), plus get_catalog_job_status (polls start_ingestion's job, a separate id space from the existing Edge-job get_job_status). Also: documents configure_database's undocumented capability prerequisite and rewords its discovery error messages accordingly, rejects empty ownerIds early instead of surfacing a raw backend error code, adds a query filter to list_capability_types so it no longer returns every type's full manifest unfiltered, exposes registerSourceTags/ingestSemanticViews/registerDataUsageStatistics on configure_database's schema sync rules, and removes create_connection's now-redundant driver-jar-specific upload convenience. Reworks upload_file's large-file handling: contentBase64 has a hard ceiling since content must be generated by the calling model as a tool argument. Adds a filePath input for reading files directly off local disk, gated at registration time (chip.ServerToolConfig.AllowLocalFileUpload, wired only through cmd/chip's own config) rather than a runtime check, so chip-service's shared registration code has no path that could ever expose it. Documents the full reasoning, including the ruled-out alternatives, in the package doc comment. --- cmd/chip/config.go | 24 ++- cmd/chip/main.go | 11 +- pkg/chip/server.go | 12 +- pkg/clients/catalog_database_client.go | 14 +- pkg/clients/dgc_domain_type_client.go | 57 +++++++ pkg/clients/dgc_domain_type_client_test.go | 39 +++++ pkg/clients/dgc_job_client.go | 46 +++++ pkg/clients/dgc_job_client_test.go | 33 ++++ pkg/clients/dgc_user_client.go | 82 +++++++++ pkg/clients/dgc_user_client_test.go | 61 +++++++ pkg/clients/edge_connection_client.go | 45 +++++ pkg/tools/configure_database/tool.go | 33 ++-- pkg/tools/create_connection/tool.go | 46 +---- pkg/tools/create_connection/tool_test.go | 34 ++-- pkg/tools/create_domain/tool.go | 2 +- pkg/tools/find_connections/tool.go | 56 ++++++ pkg/tools/find_connections/tool_test.go | 47 +++++ pkg/tools/find_domain_types/tool.go | 70 ++++++++ pkg/tools/find_domain_types/tool_test.go | 39 +++++ pkg/tools/find_users/tool.go | 80 +++++++++ pkg/tools/find_users/tool_test.go | 65 +++++++ pkg/tools/get_catalog_job_status/tool.go | 65 +++++++ pkg/tools/get_catalog_job_status/tool_test.go | 41 +++++ pkg/tools/list_capability_types/tool.go | 47 ++++- pkg/tools/list_capability_types/tool_test.go | 65 +++++++ pkg/tools/register.go | 14 +- pkg/tools/start_ingestion/tool.go | 4 +- pkg/tools/upload_file/tool.go | 161 ++++++++++++------ pkg/tools/upload_file/tool_test.go | 81 ++++++--- 29 files changed, 1200 insertions(+), 174 deletions(-) create mode 100644 pkg/clients/dgc_domain_type_client.go create mode 100644 pkg/clients/dgc_domain_type_client_test.go create mode 100644 pkg/clients/dgc_job_client.go create mode 100644 pkg/clients/dgc_job_client_test.go create mode 100644 pkg/clients/dgc_user_client.go create mode 100644 pkg/clients/dgc_user_client_test.go create mode 100644 pkg/tools/find_connections/tool.go create mode 100644 pkg/tools/find_connections/tool_test.go create mode 100644 pkg/tools/find_domain_types/tool.go create mode 100644 pkg/tools/find_domain_types/tool_test.go create mode 100644 pkg/tools/find_users/tool.go create mode 100644 pkg/tools/find_users/tool_test.go create mode 100644 pkg/tools/get_catalog_job_status/tool.go create mode 100644 pkg/tools/get_catalog_job_status/tool_test.go diff --git a/cmd/chip/config.go b/cmd/chip/config.go index 039c1fd..62f48e4 100644 --- a/cmd/chip/config.go +++ b/cmd/chip/config.go @@ -94,6 +94,11 @@ func initConfigOptions() { _ = viper.BindPFlag("mcp.enable-debug-tools", pflag.Lookup("enable-debug-tools")) viper.SetDefault("mcp.enable-debug-tools", false) + pflag.Bool("allow-local-file-upload", false, "Allow upload_file to read files directly from this machine's local disk (filePath input); off by default. Only meaningful when running chip locally against your own files — never enable this for a shared/remote deployment (env: COLLIBRA_MCP_ALLOW_LOCAL_FILE_UPLOAD)") + _ = viper.BindEnv("mcp.allow-local-file-upload", "COLLIBRA_MCP_ALLOW_LOCAL_FILE_UPLOAD") + _ = viper.BindPFlag("mcp.allow-local-file-upload", pflag.Lookup("allow-local-file-upload")) + viper.SetDefault("mcp.allow-local-file-upload", false) + pflag.StringSlice("experimental", []string{}, "Comma-separated list of opt-in experimental features to enable (env: COLLIBRA_MCP_EXPERIMENTAL). See EXPERIMENTAL FEATURES below for valid names.") _ = viper.BindEnv("mcp.experimental", "COLLIBRA_MCP_EXPERIMENTAL") _ = viper.BindPFlag("mcp.experimental", pflag.Lookup("experimental")) @@ -128,6 +133,7 @@ ENVIRONMENT VARIABLES: COLLIBRA_MCP_ENABLED_TOOLS Optional comma-separated list of tool names to enable instead of enabling all tools, cannot be used with disabled-tools COLLIBRA_MCP_DISABLED_TOOLS Optional comma-separated list of tool names to disable while enabling the remaining tools, cannot be used with enabled-tools COLLIBRA_MCP_ENABLE_DEBUG_TOOLS Enable debug tools (default: false) + COLLIBRA_MCP_ALLOW_LOCAL_FILE_UPLOAD Allow upload_file to read files from local disk (default: false); local/personal use only, never for a shared deployment COLLIBRA_MCP_EXPERIMENTAL Comma-separated list of opt-in experimental features to enable (see EXPERIMENTAL FEATURES below) COLLIBRA_MCP_SKILLS_DIR Optional path to an external skills directory merged on top of the embedded catalog (requires the 'skills' experimental feature) @@ -162,6 +168,7 @@ CONFIGURATION FILE EXAMPLE: # - "tool3" # - "tool4" enable-debug-tools: false # Optional: enable debug tools (default: false) + allow-local-file-upload: false # Optional: allow upload_file to read local files (default: false; local use only) # experimental: # Optional: opt-in experimental features (off by default) # - "skills" # skills-dir: "/path/to/skills" # Optional: external skills dir (requires the 'skills' experimental feature) @@ -218,14 +225,15 @@ type CollibraApiConfig struct { // ServerConfig holds server configuration type McpConfig struct { - Mode string `mapstructure:"mode"` // "stdio", "http", "http-sse", or "http-streamable" - Http HttpConfig `mapstructure:"http"` - Stdio StdioConfig `mapstructure:"stdio"` - EnabledTools []string `mapstructure:"enabled-tools"` - DisabledTools []string `mapstructure:"disabled-tools"` - EnableDebugTools bool `mapstructure:"enable-debug-tools"` - Experimental []string `mapstructure:"experimental"` - SkillsDir string `mapstructure:"skills-dir"` + Mode string `mapstructure:"mode"` // "stdio", "http", "http-sse", or "http-streamable" + Http HttpConfig `mapstructure:"http"` + Stdio StdioConfig `mapstructure:"stdio"` + EnabledTools []string `mapstructure:"enabled-tools"` + DisabledTools []string `mapstructure:"disabled-tools"` + EnableDebugTools bool `mapstructure:"enable-debug-tools"` + AllowLocalFileUpload bool `mapstructure:"allow-local-file-upload"` + Experimental []string `mapstructure:"experimental"` + SkillsDir string `mapstructure:"skills-dir"` } type HttpConfig struct { diff --git a/cmd/chip/main.go b/cmd/chip/main.go index 8e4856a..331f138 100644 --- a/cmd/chip/main.go +++ b/cmd/chip/main.go @@ -31,11 +31,12 @@ func main() { client := newCollibraClient(config) toolConfig := &chip.ServerToolConfig{ - EnabledTools: config.Mcp.EnabledTools, - DisabledTools: config.Mcp.DisabledTools, - EnableDebugTools: config.Mcp.EnableDebugTools, - Experimental: config.Mcp.Experimental, - SkillsDir: config.Mcp.SkillsDir, + EnabledTools: config.Mcp.EnabledTools, + DisabledTools: config.Mcp.DisabledTools, + EnableDebugTools: config.Mcp.EnableDebugTools, + AllowLocalFileUpload: config.Mcp.AllowLocalFileUpload, + Experimental: config.Mcp.Experimental, + SkillsDir: config.Mcp.SkillsDir, } serverOpts := []chip.ServerOption{ diff --git a/pkg/chip/server.go b/pkg/chip/server.go index 543b64a..6cabe71 100644 --- a/pkg/chip/server.go +++ b/pkg/chip/server.go @@ -117,7 +117,17 @@ type ServerToolConfig struct { DisabledTools []string // EnableDebugTools, when true, registers debug tools that are otherwise hidden. EnableDebugTools bool - Experimental []string + // AllowLocalFileUpload, when true, registers upload_file with its filePath + // input (reads a file directly off this process's local disk). Only meaningful + // for a chip instance running locally with access to the caller's own + // filesystem (the standalone chip binary) — a shared/remote deployment (e.g. + // chip-service) must never set this, since there "local disk" means the + // server container's disk, not any user's machine. There is deliberately no + // environment-variable escape hatch checked at call time for this — it is + // gated purely by whether the binary's own startup wiring passes true here, + // so chip-service's code has no path that could turn it on even by mistake. + AllowLocalFileUpload bool + Experimental []string // SkillsDir is the optional path to an external skills directory whose // contents are merged on top of the embedded catalog. Empty means the // embedded catalog alone is served. Only consulted when the "skills" diff --git a/pkg/clients/catalog_database_client.go b/pkg/clients/catalog_database_client.go index 81c3010..0a0c3ca 100644 --- a/pkg/clients/catalog_database_client.go +++ b/pkg/clients/catalog_database_client.go @@ -50,12 +50,16 @@ type AddDatabaseRequest struct { Description string `json:"description,omitempty"` } -// MetadataSynchronizationRule defines which tables of a schema get synchronized. +// MetadataSynchronizationRule defines which tables of a schema get synchronized and +// what additional metadata is captured for them. type MetadataSynchronizationRule struct { - Include string `json:"include,omitempty"` - Exclude string `json:"exclude,omitempty"` - TargetDomainID string `json:"targetDomainId,omitempty"` - SkipViews bool `json:"skipViews,omitempty"` + Include string `json:"include,omitempty"` + Exclude string `json:"exclude,omitempty"` + TargetDomainID string `json:"targetDomainId,omitempty"` + SkipViews bool `json:"skipViews,omitempty"` + RegisterSourceTags bool `json:"registerSourceTags,omitempty"` + IngestSemanticViews bool `json:"ingestSemanticViews,omitempty"` + RegisterDataUsageStatistics bool `json:"registerDataUsageStatistics,omitempty"` } // SchemaMetadataConfiguration links a schema connection to the synchronization rules diff --git a/pkg/clients/dgc_domain_type_client.go b/pkg/clients/dgc_domain_type_client.go new file mode 100644 index 0000000..e0a234b --- /dev/null +++ b/pkg/clients/dgc_domain_type_client.go @@ -0,0 +1,57 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// DomainType is a DGC domain type, as returned by GET /rest/2.0/domainTypes. +type DomainTypeDetails struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + PublicID string `json:"publicId,omitempty"` +} + +// DomainTypePagedResponse is the response shape for GET /rest/2.0/domainTypes. +type DomainTypePagedResponse struct { + Total int64 `json:"total"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` + Results []DomainTypeDetails `json:"results"` +} + +// FindDomainTypesQueryParams are the supported query parameters for +// GET /rest/2.0/domainTypes. Name matches case-insensitively as a substring by +// default (nameMatchMode=ANYWHERE server-side). +type FindDomainTypesQueryParams struct { + Name string `url:"name,omitempty"` + Limit int `url:"limit,omitempty"` + Offset int `url:"offset,omitempty"` +} + +// FindDomainTypes searches DGC domain types via GET /rest/2.0/domainTypes. +func FindDomainTypes(ctx context.Context, client *http.Client, params FindDomainTypesQueryParams) (*DomainTypePagedResponse, error) { + endpoint, err := buildUrl("/rest/2.0/domainTypes", params) + if err != nil { + return nil, fmt.Errorf("finding domain types: building endpoint: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("finding domain types: building request: %w", err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("finding domain types: %w", err) + } + + var result DomainTypePagedResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("finding domain types: decoding response: %w", err) + } + return &result, nil +} diff --git a/pkg/clients/dgc_domain_type_client_test.go b/pkg/clients/dgc_domain_type_client_test.go new file mode 100644 index 0000000..9345b2c --- /dev/null +++ b/pkg/clients/dgc_domain_type_client_test.go @@ -0,0 +1,39 @@ +package clients_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindDomainTypes(t *testing.T) { + typeID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/domainTypes", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("name") != "Physical Data Dictionary" { + t.Fatalf("unexpected name query param: %s", r.URL.Query().Get("name")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":1,"offset":0,"limit":50,"results":[{"id":"` + typeID.String() + `","name":"Physical Data Dictionary","publicId":"PhysicalDataDictionary"}]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + response, err := clients.FindDomainTypes(t.Context(), client, clients.FindDomainTypesQueryParams{Name: "Physical Data Dictionary", Limit: 50}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if response.Total != 1 || len(response.Results) != 1 { + t.Fatalf("unexpected response: %+v", response) + } + if response.Results[0].ID != typeID.String() { + t.Fatalf("expected id %s, got %s", typeID.String(), response.Results[0].ID) + } +} diff --git a/pkg/clients/dgc_job_client.go b/pkg/clients/dgc_job_client.go new file mode 100644 index 0000000..7478732 --- /dev/null +++ b/pkg/clients/dgc_job_client.go @@ -0,0 +1,46 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// CatalogJob is a DGC job (e.g. the job returned by +// POST /rest/catalogDatabase/v1/databases/{id}/synchronizeMetadata), as returned by +// GET /rest/jobs/v1/jobs/{jobId}. This is the DGC-wide job resource — distinct from +// edge-management's per-Edge-site jobs (see GetJobStatusLog / get_job_status), which +// track Edge capability runs and connection tests instead. +type CatalogJob struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + State string `json:"state,omitempty"` + Result string `json:"result,omitempty"` + Message string `json:"message,omitempty"` + ProgressPercentage float64 `json:"progressPercentage,omitempty"` + StartDate string `json:"startDate,omitempty"` + EndDate string `json:"endDate,omitempty"` +} + +// GetCatalogJob fetches a DGC job's status via GET /rest/jobs/v1/jobs/{jobId}. This is +// the current, non-deprecated DGC job resource (the older /rest/2.0/jobs/{jobId} is +// deprecated for removal, per its own javadoc, in favor of this one). +func GetCatalogJob(ctx context.Context, client *http.Client, jobID string) (*CatalogJob, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/rest/jobs/v1/jobs/"+jobID, nil) + if err != nil { + return nil, fmt.Errorf("getting catalog job %q: building request: %w", jobID, err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("getting catalog job %q: %w", jobID, err) + } + + var result CatalogJob + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("getting catalog job %q: decoding response: %w", jobID, err) + } + return &result, nil +} diff --git a/pkg/clients/dgc_job_client_test.go b/pkg/clients/dgc_job_client_test.go new file mode 100644 index 0000000..42dc527 --- /dev/null +++ b/pkg/clients/dgc_job_client_test.go @@ -0,0 +1,33 @@ +package clients_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestGetCatalogJob(t *testing.T) { + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/jobs/v1/jobs/"+jobID.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"` + jobID.String() + `","name":"Database Synchronisation of source","type":"DELTA_INGESTION","state":"RUNNING","result":"NOT_SET","progressPercentage":42}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + job, err := clients.GetCatalogJob(t.Context(), client, jobID.String()) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if job.State != "RUNNING" || job.Type != "DELTA_INGESTION" || job.ProgressPercentage != 42 { + t.Fatalf("unexpected job: %+v", job) + } +} diff --git a/pkg/clients/dgc_user_client.go b/pkg/clients/dgc_user_client.go new file mode 100644 index 0000000..da3f41c --- /dev/null +++ b/pkg/clients/dgc_user_client.go @@ -0,0 +1,82 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// User is a DGC user, as returned by GET /rest/2.0/users and +// GET /rest/2.0/users/{id}. +type User struct { + ID string `json:"id"` + UserName string `json:"userName,omitempty"` + FirstName string `json:"firstName,omitempty"` + LastName string `json:"lastName,omitempty"` + EmailAddress string `json:"emailAddress,omitempty"` + Title string `json:"title,omitempty"` + Department string `json:"department,omitempty"` + Enabled bool `json:"enabled"` +} + +// UserPagedResponse is the response shape for GET /rest/2.0/users. +type UserPagedResponse struct { + Total int64 `json:"total"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` + Results []User `json:"results"` +} + +// FindUsersQueryParams are the supported query parameters for GET /rest/2.0/users. +// Name matches case-insensitively as a substring, by default across username, +// first name, last name, and first+last/last+first combinations. +type FindUsersQueryParams struct { + Name string `url:"name,omitempty"` + Limit int `url:"limit,omitempty"` + Offset int `url:"offset,omitempty"` + IncludeDisabled bool `url:"includeDisabled,omitempty"` +} + +// FindUsers searches DGC users via GET /rest/2.0/users. +func FindUsers(ctx context.Context, client *http.Client, params FindUsersQueryParams) (*UserPagedResponse, error) { + endpoint, err := buildUrl("/rest/2.0/users", params) + if err != nil { + return nil, fmt.Errorf("finding users: building endpoint: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("finding users: building request: %w", err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("finding users: %w", err) + } + + var result UserPagedResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("finding users: decoding response: %w", err) + } + return &result, nil +} + +// GetUser fetches a single DGC user by id via GET /rest/2.0/users/{id}. +func GetUser(ctx context.Context, client *http.Client, userID string) (*User, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/rest/2.0/users/"+userID, nil) + if err != nil { + return nil, fmt.Errorf("getting user %q: building request: %w", userID, err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("getting user %q: %w", userID, err) + } + + var result User + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("getting user %q: decoding response: %w", userID, err) + } + return &result, nil +} diff --git a/pkg/clients/dgc_user_client_test.go b/pkg/clients/dgc_user_client_test.go new file mode 100644 index 0000000..29f7cbc --- /dev/null +++ b/pkg/clients/dgc_user_client_test.go @@ -0,0 +1,61 @@ +package clients_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindUsers(t *testing.T) { + adminID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/users", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("name") != "Admin" { + t.Fatalf("unexpected name query param: %s", r.URL.Query().Get("name")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":1,"offset":0,"limit":20,"results":[{"id":"` + adminID.String() + `","userName":"Admin","emailAddress":"admin@example.com","enabled":true}]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + response, err := clients.FindUsers(t.Context(), client, clients.FindUsersQueryParams{Name: "Admin", Limit: 20}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if response.Total != 1 || len(response.Results) != 1 { + t.Fatalf("unexpected response: %+v", response) + } + if response.Results[0].ID != adminID.String() { + t.Fatalf("expected id %s, got %s", adminID.String(), response.Results[0].ID) + } +} + +func TestGetUser(t *testing.T) { + userID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/users/"+userID.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"` + userID.String() + `","userName":"jdoe","firstName":"Jane","lastName":"Doe","enabled":true}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + user, err := clients.GetUser(t.Context(), client, userID.String()) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if user.UserName != "jdoe" || user.FirstName != "Jane" { + t.Fatalf("unexpected user: %+v", user) + } +} diff --git a/pkg/clients/edge_connection_client.go b/pkg/clients/edge_connection_client.go index 4ea4346..101a35c 100644 --- a/pkg/clients/edge_connection_client.go +++ b/pkg/clients/edge_connection_client.go @@ -45,6 +45,51 @@ func CreateOrUpdateConnection(ctx context.Context, client *http.Client, connecti return doConnectionRequest(ctx, client, http.MethodPut, endpoint, request) } +// ConnectionFindRequest is the request body for POST /edge/api/rest/v2/connections/find. +type ConnectionFindRequest struct { + EdgeSiteID string `json:"edgeSiteId,omitempty"` + Name string `json:"name,omitempty"` + NameMatchMode string `json:"nameMatchMode,omitempty"` +} + +// FindConnections searches Edge connections via POST /edge/api/rest/v2/connections/find. +// Useful for picking up a connection a user created manually (e.g. via the DGC/Edge UI, +// for a driver file too large to pass through this tool) by name, instead of needing +// its id. +func FindConnections(ctx context.Context, client *http.Client, request ConnectionFindRequest) ([]Connection, error) { + body, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("finding connections: marshaling request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "/edge/api/rest/v2/connections/find", bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("finding connections: building request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("finding connections: sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("finding connections: reading response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("finding connections: unexpected status %d: %s", resp.StatusCode, string(respBody)) + } + + var result []Connection + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("finding connections: decoding response: %w", err) + } + return result, nil +} + func doConnectionRequest(ctx context.Context, client *http.Client, method, endpoint string, request ConnectionRequest) (*Connection, error) { body, err := json.Marshal(request) if err != nil { diff --git a/pkg/tools/configure_database/tool.go b/pkg/tools/configure_database/tool.go index 5e65546..fdf5495 100644 --- a/pkg/tools/configure_database/tool.go +++ b/pkg/tools/configure_database/tool.go @@ -38,13 +38,16 @@ type Input struct { DatabaseName string `json:"databaseName,omitempty" jsonschema:"Optional. Exact name of the database (catalog) to register, as it appears at the data source. Required if the data source exposes more than one database/catalog through this connection; if there is exactly one, it is selected automatically."` CommunityID string `json:"communityId" jsonschema:"UUID of the community the Database asset (and its automatically created domain) will be created in."` ParentSystemID string `json:"parentSystemId" jsonschema:"UUID of the parent System asset the Database asset will be linked to."` - OwnerIDs []string `json:"ownerIds" jsonschema:"UUIDs of the users to assign as owners of the Database asset."` + OwnerIDs []string `json:"ownerIds" jsonschema:"UUIDs of the users to assign as owners of the Database asset. Use find_users to resolve a name (e.g. 'Admin') to its UUID."` Description string `json:"description,omitempty" jsonschema:"Optional description of the Database asset."` - Include string `json:"include,omitempty" jsonschema:"Optional. Comma-separated table name pattern to synchronize, '*' wildcard supported. Defaults to '*' (all tables)."` - Exclude string `json:"exclude,omitempty" jsonschema:"Optional. Comma-separated table name pattern to exclude from synchronization."` - TargetDomainID string `json:"targetDomainId,omitempty" jsonschema:"Optional. UUID of a domain to create synchronized assets in. If omitted, an automatically created domain per schema is used."` - SkipViews bool `json:"skipViews,omitempty" jsonschema:"Optional. If true, database views are excluded from synchronization."` + Include string `json:"include,omitempty" jsonschema:"Optional. Comma-separated table name pattern to synchronize, '*' wildcard supported. Defaults to '*' (all tables)."` + Exclude string `json:"exclude,omitempty" jsonschema:"Optional. Comma-separated table name pattern to exclude from synchronization."` + TargetDomainID string `json:"targetDomainId,omitempty" jsonschema:"Optional. UUID of a domain to create synchronized assets in. If omitted, an automatically created domain per schema is used."` + SkipViews bool `json:"skipViews,omitempty" jsonschema:"Optional. If true, database views are excluded from synchronization."` + RegisterSourceTags bool `json:"registerSourceTags,omitempty" jsonschema:"Optional. If true, registers tags from the data source (when supported by the driver)."` + IngestSemanticViews bool `json:"ingestSemanticViews,omitempty" jsonschema:"Optional. If true, ingests semantic views (when supported by the driver)."` + RegisterDataUsageStatistics bool `json:"registerDataUsageStatistics,omitempty" jsonschema:"Optional. If true, calculates data usage statistics for the synchronized tables (when supported by the data source)."` } type Output struct { @@ -58,7 +61,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "configure_database", Title: "Configure Database for Ingestion", - Description: "Discovers a database through an Edge connection, registers it as a Database asset, and configures which tables get synchronized. Prerequisite for start_ingestion. Assumes the target community, and parent System asset already exist.", + Description: "Discovers a database through an Edge connection, registers it as a Database asset, and configures which tables get synchronized. Prerequisite for start_ingestion. Assumes the target community and parent System asset already exist (create_community/create_domain/create_asset), and that a jdbc-ingestion capability referencing this connection has already been created via create_capability — the discovery/refresh steps here only find data because that capability actually performs the crawl; without one, this fails with a discovery-timeout-shaped error even though the real cause is the missing capability.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, @@ -76,6 +79,9 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { if err := validation.UUID("parentSystemId", input.ParentSystemID); err != nil { return Output{}, err } + if len(input.OwnerIDs) == 0 { + return Output{}, fmt.Errorf("ownerIds must contain at least one user UUID — use find_users to resolve a name to an id") + } if err := validation.UUIDs("ownerIds", input.OwnerIDs); err != nil { return Output{}, err } @@ -115,10 +121,13 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { SchemaConnectionID: schemaConnection.ID, SynchronizationRules: []clients.MetadataSynchronizationRule{ { - Include: include, - Exclude: input.Exclude, - TargetDomainID: input.TargetDomainID, - SkipViews: input.SkipViews, + Include: include, + Exclude: input.Exclude, + TargetDomainID: input.TargetDomainID, + SkipViews: input.SkipViews, + RegisterSourceTags: input.RegisterSourceTags, + IngestSemanticViews: input.IngestSemanticViews, + RegisterDataUsageStatistics: input.RegisterDataUsageStatistics, }, }, } @@ -157,7 +166,7 @@ func discoverDatabaseConnection(ctx context.Context, client *http.Client, edgeCo } if len(connections) == 0 { - return nil, fmt.Errorf("no database connections were discovered for edge connection %s after %d attempts; the refresh may still be in progress — retry this tool call", edgeConnectionID, pollAttempts) + return nil, fmt.Errorf("no database connections were discovered for edge connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or — more commonly — no capability referencing this connection exists yet: the discovery/refresh only finds data because a jdbc-ingestion capability actually performs the crawl. Verify a capability exists for this connection (create_capability) before retrying", edgeConnectionID, pollAttempts) } if databaseName != "" { @@ -196,5 +205,5 @@ func discoverSchemaConnections(ctx context.Context, client *http.Client, databas } } - return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts; the refresh may still be in progress — retry this tool call", databaseConnectionID, pollAttempts) + return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or no capability referencing this connection exists yet — verify with create_capability", databaseConnectionID, pollAttempts) } diff --git a/pkg/tools/create_connection/tool.go b/pkg/tools/create_connection/tool.go index 25250df..eec432f 100644 --- a/pkg/tools/create_connection/tool.go +++ b/pkg/tools/create_connection/tool.go @@ -6,7 +6,6 @@ package create_connection import ( "context" "fmt" - "io" "net/http" "github.com/collibra/chip/pkg/chip" @@ -22,10 +21,7 @@ type Input struct { TypeID string `json:"typeId" jsonschema:"The id of the connection type (e.g. 'Generic' for a generic JDBC connection, or a vendor-specific type such as a Snowflake connection type). Determines which parameters are expected — use list_capability_types to inspect a type's manifest."` EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this connection will be valid. Use the list_edge_sites tool to discover available sites."` VaultID string `json:"vaultId,omitempty" jsonschema:"Optional UUID of the vault to retrieve vault-backed parameters from."` - Parameters map[string]any `json:"parameters" jsonschema:"Fixed connection parameters as defined by the connection type's manifest (e.g. driver-class, connection-string for the 'Generic' JDBC connection type). If driverJarUrl is provided, the driver-jar parameter is set automatically and does not need to be included here. Do NOT put open-ended/vendor-specific properties (e.g. a Snowflake connection's Role, Warehouse, User, Database, or a private key file) here — use additionalProperties instead. If you don't already know a data source's driver class, connection string format, or required properties, call get_data_source_setup_guide first instead of guessing."` - - DriverJarURL string `json:"driverJarUrl,omitempty" jsonschema:"Optional. A URL to download a JDBC driver jar from (e.g. a Maven Central artifact URL). If provided, the jar is downloaded and uploaded to the edge site, and the resulting artifact URI is set as the 'driver-jar' connection parameter."` - DriverJarFilename string `json:"driverJarFilename,omitempty" jsonschema:"Required if driverJarUrl is provided. The filename to upload the driver jar as (e.g. 'postgresql-42.7.11.jar')."` + Parameters map[string]any `json:"parameters" jsonschema:"Fixed connection parameters as defined by the connection type's manifest (e.g. driver-class, connection-string for the 'Generic' JDBC connection type). Do NOT put open-ended/vendor-specific properties (e.g. a Snowflake connection's Role, Warehouse, User, Database, or a private key file) here — use additionalProperties instead. If you don't already know a data source's driver class, connection string format, required properties, or verified driver source, call get_data_source_setup_guide first instead of guessing. For 'driver-jar' (and any other FILE-type parameter), the value is an artifact URI from upload_file — see that tool's policy on where driver files may come from."` AdditionalProperties []AdditionalProperty `json:"additionalProperties,omitempty" jsonschema:"Optional. Open-ended, connection-type-defined properties beyond the fixed manifest parameters — e.g. a Snowflake connection (via the 'Generic' JDBC connection type) uses this for Role, Warehouse, User, Database, and private_key_file. Confirmed shape: each entry is injected as {name, type, value, secret} into a single array-valued parameter (see additionalPropertiesKey). For a FILE-type property, first call upload_file and use its returned artifact URI as this entry's value with type='file'."` AdditionalPropertiesKey string `json:"additionalPropertiesKey,omitempty" jsonschema:"Optional. The manifest parameter name that additionalProperties are grouped under. Defaults to 'connection-properties', which is correct for the 'Generic' JDBC connection type (confirmed live). Other connection types may use a different key (e.g. AWS/GCP connections use 'additional-parameters') — check the type's manifest via list_capability_types if unsure."` @@ -53,7 +49,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "create_connection", Title: "Create or Update Edge Connection", - Description: "Creates or updates an Edge connection (e.g. a JDBC connection for the jdbc-ingestion capability) via the private Edge connection management API.", + Description: "Creates or updates an Edge connection (e.g. a JDBC connection for the jdbc-ingestion capability) via the private Edge connection management API. Does not download driver files itself — see upload_file for how driver jars must be sourced (Collibra Marketplace only, not arbitrary URLs).", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, @@ -77,17 +73,6 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { parameters = map[string]any{} } - if input.DriverJarURL != "" { - if input.DriverJarFilename == "" { - return Output{}, fmt.Errorf("driverJarFilename is required when driverJarUrl is provided") - } - uri, err := downloadAndUploadDriver(ctx, collibraClient, input.DriverJarURL, input.DriverJarFilename) - if err != nil { - return Output{Success: false, Error: fmt.Sprintf("failed to prepare driver jar: %s", err.Error())}, nil - } - parameters["driver-jar"] = uri - } - if len(input.AdditionalProperties) > 0 { key := input.AdditionalPropertiesKey if key == "" { @@ -135,30 +120,3 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { return Output{Connection: connection, Success: true}, nil } } - -// downloadAndUploadDriver fetches the driver jar bytes from an arbitrary external URL -// (not routed through the Collibra client, since it points outside the tenant) and -// re-uploads them to the edge site via the authenticated Collibra client. -func downloadAndUploadDriver(ctx context.Context, collibraClient *http.Client, url, filename string) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return "", fmt.Errorf("building driver download request: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", fmt.Errorf("downloading driver jar: %w", err) - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("downloading driver jar: unexpected status %d", resp.StatusCode) - } - - content, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("reading driver jar content: %w", err) - } - - return clients.UploadFile(ctx, collibraClient, filename, content) -} diff --git a/pkg/tools/create_connection/tool_test.go b/pkg/tools/create_connection/tool_test.go index 4aca41c..7d21b96 100644 --- a/pkg/tools/create_connection/tool_test.go +++ b/pkg/tools/create_connection/tool_test.go @@ -182,18 +182,16 @@ func TestCreateConnection_AdditionalProperties_CustomKey(t *testing.T) { } } -func TestCreateConnection_DriverJarUpload(t *testing.T) { +// create_connection deliberately does not fetch driver files from arbitrary URLs +// (Collibra Marketplace requires an authenticated download — see upload_file's +// policy doc) — the only supported path is referencing an artifact URI that was +// already uploaded via upload_file, passed through like any other parameter. +func TestCreateConnection_PreUploadedDriverJar(t *testing.T) { siteID, _ := uuid.NewUUID() connID, _ := uuid.NewUUID() + const uploadedJarURI = "jar://test-uuid/driver.jar" - var uploadedJarURI string handler := http.NewServeMux() - handler.HandleFunc("POST /edge/api/rest/v2/upload", func(w http.ResponseWriter, r *http.Request) { - uploadedJarURI = "jar://test-uuid/driver.jar" - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`"` + uploadedJarURI + `"`)) - }) handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { if in.Parameters["driver-jar"] != uploadedJarURI { t.Fatalf("expected driver-jar %q, got %v", uploadedJarURI, in.Parameters["driver-jar"]) @@ -206,21 +204,15 @@ func TestCreateConnection_DriverJarUpload(t *testing.T) { } })) - edgeServer := httptest.NewServer(handler) - defer edgeServer.Close() - - driverServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte("fake jar bytes")) - })) - defer driverServer.Close() + server := httptest.NewServer(handler) + defer server.Close() - client := testutil.NewClient(edgeServer) + client := testutil.NewClient(server) output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ - Name: "local-postgres-source", - TypeID: "Generic", - EdgeSiteID: siteID.String(), - DriverJarURL: driverServer.URL, - DriverJarFilename: "driver.jar", + Name: "local-postgres-source", + TypeID: "Generic", + EdgeSiteID: siteID.String(), + Parameters: map[string]any{"driver-jar": uploadedJarURI}, }) if err != nil { t.Fatalf("expected no error, got: %v", err) diff --git a/pkg/tools/create_domain/tool.go b/pkg/tools/create_domain/tool.go index 1018ac0..e32d25c 100644 --- a/pkg/tools/create_domain/tool.go +++ b/pkg/tools/create_domain/tool.go @@ -18,7 +18,7 @@ type Input struct { Name string `json:"name" jsonschema:"The domain name."` Description string `json:"description,omitempty" jsonschema:"Optional description of the domain."` CommunityID string `json:"communityId" jsonschema:"UUID of the community (created via create_community) this domain belongs to."` - TypeID string `json:"typeId" jsonschema:"UUID of the domain type. There is no discovery endpoint for this in DGC's public API; the well-known 'Physical Data Dictionary' domain type id (00000000-0000-0000-0000-000000030011), used for jdbc-ingestion databases, is stable across DGC versions."` + TypeID string `json:"typeId" jsonschema:"UUID of the domain type. Use find_domain_types to resolve a name (e.g. 'Physical Data Dictionary', used for jdbc-ingestion databases) to its UUID."` ExcludedFromAutoHyperlinking bool `json:"excludedFromAutoHyperlinking,omitempty" jsonschema:"Optional. If true, assets in this domain are excluded from automatic hyperlinking."` } diff --git a/pkg/tools/find_connections/tool.go b/pkg/tools/find_connections/tool.go new file mode 100644 index 0000000..74e336e --- /dev/null +++ b/pkg/tools/find_connections/tool.go @@ -0,0 +1,56 @@ +// Package find_connections implements the find_connections MCP tool: finds Edge +// connections by name. Primarily for picking up a connection the user created +// manually (e.g. via the DGC/Edge UI, when a driver file was too large to upload +// through upload_file's contentBase64) — see create_connection's package doc for the +// guided-manual-creation flow this supports. +package find_connections + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + EdgeSiteID string `json:"edgeSiteId,omitempty" jsonschema:"Optional. UUID of the edge site to restrict the search to."` + Name string `json:"name,omitempty" jsonschema:"Optional. The connection name to search for."` + NameMatchMode string `json:"nameMatchMode,omitempty" jsonschema:"Optional. 'ANYWHERE' (default, substring match) or 'EXACT'."` +} + +type Output struct { + Connections []clients.Connection `json:"connections" jsonschema:"The matching connections."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "find_connections", + Title: "Find Edge Connections", + Description: "Finds Edge connections by name (and optionally edge site). Use this to pick up a connection that already exists — e.g. one the user created manually via the DGC/Edge UI for a driver file too large to pass through upload_file — instead of creating a duplicate with create_connection.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUIDOptional("edgeSiteId", input.EdgeSiteID); err != nil { + return Output{}, err + } + + connections, err := clients.FindConnections(ctx, collibraClient, clients.ConnectionFindRequest{ + EdgeSiteID: input.EdgeSiteID, + Name: input.Name, + NameMatchMode: input.NameMatchMode, + }) + if err != nil { + return Output{}, err + } + + return Output{Connections: connections}, nil + } +} diff --git a/pkg/tools/find_connections/tool_test.go b/pkg/tools/find_connections/tool_test.go new file mode 100644 index 0000000..8deef26 --- /dev/null +++ b/pkg/tools/find_connections/tool_test.go @@ -0,0 +1,47 @@ +package find_connections_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/find_connections" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindConnections(t *testing.T) { + connID, _ := uuid.NewUUID() + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("POST /edge/api/rest/v2/connections/find", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionFindRequest) (int, []clients.Connection) { + if in.Name != "snowflake-manual" { + t.Fatalf("unexpected name: %s", in.Name) + } + return http.StatusOK, []clients.Connection{ + {ID: connID.String(), Name: "snowflake-manual", EdgeSiteID: siteID.String()}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Name: "snowflake-manual"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.Connections) != 1 || output.Connections[0].ID != connID.String() { + t.Fatalf("unexpected output: %+v", output) + } +} + +func TestFindConnections_InvalidEdgeSiteID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid edgeSiteId") + } +} diff --git a/pkg/tools/find_domain_types/tool.go b/pkg/tools/find_domain_types/tool.go new file mode 100644 index 0000000..a34ac04 --- /dev/null +++ b/pkg/tools/find_domain_types/tool.go @@ -0,0 +1,70 @@ +// Package find_domain_types implements the find_domain_types MCP tool: resolves a +// domain type's name (e.g. "Physical Data Dictionary") to its UUID, for +// create_domain's typeId — there is no other discovery path for this, since domain +// types are not returned by list_capability_types or any Edge API. +package find_domain_types + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name,omitempty" jsonschema:"Optional. Matches case-insensitively as a substring against the domain type name (e.g. 'Physical Data Dictionary', 'Technology Asset'). Omit to list all domain types."` + Limit int `json:"limit,omitempty" jsonschema:"Optional. Maximum number of results to return. Default: 50."` + Offset int `json:"offset,omitempty" jsonschema:"Optional. Index of first result (pagination offset). Default: 0."` +} + +type Output struct { + Total int64 `json:"total" jsonschema:"The total number of domain types matching the search criteria."` + Offset int64 `json:"offset" jsonschema:"The offset for the results."` + Limit int64 `json:"limit" jsonschema:"The maximum number of results returned."` + DomainTypes []DomainType `json:"domainTypes" jsonschema:"The matching domain types."` +} + +type DomainType struct { + ID string `json:"id" jsonschema:"The domain type's UUID — use this for create_domain's typeId."` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + PublicID string `json:"publicId,omitempty"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "find_domain_types", + Title: "Find Domain Types", + Description: "Finds DGC domain types by name (substring, case-insensitive), returning their UUIDs. Use this to resolve a domain type name (e.g. 'Physical Data Dictionary', needed for jdbc-ingestion databases) to a typeId for create_domain, instead of needing to already know the UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + limit := input.Limit + if limit == 0 { + limit = 50 + } + + response, err := clients.FindDomainTypes(ctx, collibraClient, clients.FindDomainTypesQueryParams{ + Name: input.Name, + Limit: limit, + Offset: input.Offset, + }) + if err != nil { + return Output{}, err + } + + domainTypes := make([]DomainType, len(response.Results)) + for i, d := range response.Results { + domainTypes[i] = DomainType{ID: d.ID, Name: d.Name, Description: d.Description, PublicID: d.PublicID} + } + + return Output{Total: response.Total, Offset: response.Offset, Limit: response.Limit, DomainTypes: domainTypes}, nil + } +} diff --git a/pkg/tools/find_domain_types/tool_test.go b/pkg/tools/find_domain_types/tool_test.go new file mode 100644 index 0000000..44d13d3 --- /dev/null +++ b/pkg/tools/find_domain_types/tool_test.go @@ -0,0 +1,39 @@ +package find_domain_types_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/find_domain_types" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindDomainTypes(t *testing.T) { + typeID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/domainTypes", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("limit") != "50" { + t.Fatalf("expected default limit 50, got: %s", r.URL.Query().Get("limit")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":1,"offset":0,"limit":50,"results":[{"id":"` + typeID.String() + `","name":"Physical Data Dictionary"}]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Name: "Physical"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Total != 1 || len(output.DomainTypes) != 1 { + t.Fatalf("unexpected output: %+v", output) + } + if output.DomainTypes[0].ID != typeID.String() { + t.Fatalf("expected id %s, got %s", typeID.String(), output.DomainTypes[0].ID) + } +} diff --git a/pkg/tools/find_users/tool.go b/pkg/tools/find_users/tool.go new file mode 100644 index 0000000..2066bf4 --- /dev/null +++ b/pkg/tools/find_users/tool.go @@ -0,0 +1,80 @@ +// Package find_users implements the find_users MCP tool: resolves a user's name +// (or partial name) to their UUID, e.g. to fill in configure_database's ownerIds +// without the caller needing to already know the id. +package find_users + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name,omitempty" jsonschema:"Optional. Matches case-insensitively as a substring against username, first name, last name, and first+last name combinations (e.g. 'admin', 'jane', 'jane doe'). Omit to list users."` + IncludeDisabled bool `json:"includeDisabled,omitempty" jsonschema:"Optional. If true, includes disabled user accounts. Defaults to false (enabled users only)."` + Limit int `json:"limit,omitempty" jsonschema:"Optional. Maximum number of results to return. Default: 20."` + Offset int `json:"offset,omitempty" jsonschema:"Optional. Index of first result (pagination offset). Default: 0."` +} + +type Output struct { + Total int64 `json:"total" jsonschema:"The total number of users matching the search criteria."` + Offset int64 `json:"offset" jsonschema:"The offset for the results."` + Limit int64 `json:"limit" jsonschema:"The maximum number of results returned."` + Users []User `json:"users" jsonschema:"The matching users."` +} + +type User struct { + ID string `json:"id" jsonschema:"The user's UUID — use this for ownerIds, createdBy, etc."` + UserName string `json:"userName,omitempty" jsonschema:"The user's username."` + FirstName string `json:"firstName,omitempty"` + LastName string `json:"lastName,omitempty"` + EmailAddress string `json:"emailAddress,omitempty"` + Enabled bool `json:"enabled" jsonschema:"Whether the user account is enabled."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "find_users", + Title: "Find Users", + Description: "Finds DGC users by name (substring, case-insensitive), returning their UUIDs. Use this to resolve a name to an id for configure_database's ownerIds or similar owner/assignee fields, instead of needing to already know the UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + limit := input.Limit + if limit == 0 { + limit = 20 + } + + response, err := clients.FindUsers(ctx, collibraClient, clients.FindUsersQueryParams{ + Name: input.Name, + Limit: limit, + Offset: input.Offset, + IncludeDisabled: input.IncludeDisabled, + }) + if err != nil { + return Output{}, err + } + + users := make([]User, len(response.Results)) + for i, u := range response.Results { + users[i] = User{ + ID: u.ID, + UserName: u.UserName, + FirstName: u.FirstName, + LastName: u.LastName, + EmailAddress: u.EmailAddress, + Enabled: u.Enabled, + } + } + + return Output{Total: response.Total, Offset: response.Offset, Limit: response.Limit, Users: users}, nil + } +} diff --git a/pkg/tools/find_users/tool_test.go b/pkg/tools/find_users/tool_test.go new file mode 100644 index 0000000..9ed377f --- /dev/null +++ b/pkg/tools/find_users/tool_test.go @@ -0,0 +1,65 @@ +package find_users_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/find_users" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestFindUsers(t *testing.T) { + adminID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/users", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("name") != "Admin" { + t.Fatalf("unexpected name query param: %s", r.URL.Query().Get("name")) + } + if r.URL.Query().Get("limit") != "20" { + t.Fatalf("expected default limit 20, got: %s", r.URL.Query().Get("limit")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":1,"offset":0,"limit":20,"results":[{"id":"` + adminID.String() + `","userName":"Admin","emailAddress":"admin@example.com","enabled":true}]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Name: "Admin"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Total != 1 || len(output.Users) != 1 { + t.Fatalf("unexpected output: %+v", output) + } + if output.Users[0].ID != adminID.String() { + t.Fatalf("expected id %s, got %s", adminID.String(), output.Users[0].ID) + } +} + +func TestFindUsers_CustomLimit(t *testing.T) { + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/2.0/users", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("limit") != "5" { + t.Fatalf("expected limit 5, got: %s", r.URL.Query().Get("limit")) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"total":0,"offset":0,"limit":5,"results":[]}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Limit: 5}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.Users) != 0 { + t.Fatalf("expected no users, got: %+v", output.Users) + } +} diff --git a/pkg/tools/get_catalog_job_status/tool.go b/pkg/tools/get_catalog_job_status/tool.go new file mode 100644 index 0000000..db4636a --- /dev/null +++ b/pkg/tools/get_catalog_job_status/tool.go @@ -0,0 +1,65 @@ +// Package get_catalog_job_status implements the get_catalog_job_status MCP tool: +// polls the status of a DGC catalog job, e.g. the one returned by start_ingestion. +// This is distinct from get_job_status, which polls Edge-site jobs (capability runs, +// connection tests) — the two job id spaces are not interchangeable. +package get_catalog_job_status + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + JobID string `json:"jobId" jsonschema:"UUID of the DGC catalog job to check, as returned by start_ingestion. Do not pass an Edge job id here (e.g. from test_connection) — use get_job_status for those instead."` +} + +type Output struct { + Name string `json:"name,omitempty" jsonschema:"The job's name."` + Type string `json:"type,omitempty" jsonschema:"The job's type (e.g. 'DELTA_INGESTION')."` + State string `json:"state,omitempty" jsonschema:"The job's current state: WAITING, RUNNING, CANCELING, COMPLETED, CANCELED, or ERROR."` + Result string `json:"result,omitempty" jsonschema:"The job's result once finished: NOT_SET (not finished yet), SUCCESS, COMPLETED_WITH_ERROR, FAILURE, or ABORTED."` + Message string `json:"message,omitempty" jsonschema:"Status message from the job."` + ProgressPercentage float64 `json:"progressPercentage,omitempty" jsonschema:"Progress percentage, 0-100."` + StartDate string `json:"startDate,omitempty"` + EndDate string `json:"endDate,omitempty" jsonschema:"When the job finished, if it has."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_catalog_job_status", + Title: "Get Catalog Job Status", + Description: "Gets the current status of a DGC catalog job, e.g. the ingestion job started by start_ingestion. For Edge-site jobs (test_connection, capability runs), use get_job_status instead — the two job id spaces are separate and not interchangeable.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("jobId", input.JobID); err != nil { + return Output{}, err + } + + job, err := clients.GetCatalogJob(ctx, collibraClient, input.JobID) + if err != nil { + return Output{}, err + } + + return Output{ + Name: job.Name, + Type: job.Type, + State: job.State, + Result: job.Result, + Message: job.Message, + ProgressPercentage: job.ProgressPercentage, + StartDate: job.StartDate, + EndDate: job.EndDate, + }, nil + } +} diff --git a/pkg/tools/get_catalog_job_status/tool_test.go b/pkg/tools/get_catalog_job_status/tool_test.go new file mode 100644 index 0000000..a5840c9 --- /dev/null +++ b/pkg/tools/get_catalog_job_status/tool_test.go @@ -0,0 +1,41 @@ +package get_catalog_job_status_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/get_catalog_job_status" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestGetCatalogJobStatus(t *testing.T) { + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.HandleFunc("GET /rest/jobs/v1/jobs/"+jobID.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"` + jobID.String() + `","name":"Database Synchronisation of source","type":"DELTA_INGESTION","state":"COMPLETED","result":"SUCCESS","progressPercentage":100}`)) + }) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: jobID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.State != "COMPLETED" || output.Result != "SUCCESS" { + t.Fatalf("unexpected output: %+v", output) + } +} + +func TestGetCatalogJobStatus_InvalidJobID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid jobId") + } +} diff --git a/pkg/tools/list_capability_types/tool.go b/pkg/tools/list_capability_types/tool.go index 985f86f..016c3ac 100644 --- a/pkg/tools/list_capability_types/tool.go +++ b/pkg/tools/list_capability_types/tool.go @@ -6,6 +6,7 @@ package list_capability_types import ( "context" "net/http" + "strings" "github.com/collibra/chip/pkg/chip" "github.com/collibra/chip/pkg/clients" @@ -15,18 +16,19 @@ import ( type Input struct { EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site to list available capability and connection types for. Use the list_edge_sites tool to discover available sites."` + Query string `json:"query,omitempty" jsonschema:"Optional. Case-insensitive substring filter on type id (e.g. 'jdbc', 'snowflake'). Without a query, all type ids are returned WITHOUT their manifest (an edge site can have 80+ types; manifests are large) — call again with a query matching the type you want to see its full manifest with expected parameters."` } type Output struct { - CapabilityTypes []clients.CapabilityType `json:"capabilityTypes" jsonschema:"Capability types available on this edge site (e.g. 'jdbc-ingestion'), with their manifests describing expected install parameters."` - ConnectionTypes []clients.ConnectionType `json:"connectionTypes" jsonschema:"Connection types available on this edge site (e.g. 'Generic' or a vendor-specific type), with their manifests describing expected parameters."` + CapabilityTypes []clients.CapabilityType `json:"capabilityTypes" jsonschema:"Capability types available on this edge site (e.g. 'jdbc-ingestion'). Manifest is only populated when query narrows the results — see query's description."` + ConnectionTypes []clients.ConnectionType `json:"connectionTypes" jsonschema:"Connection types available on this edge site (e.g. 'Generic' or a vendor-specific type). Manifest is only populated when query narrows the results — see query's description."` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "list_capability_types", Title: "List Capability and Connection Types", - Description: "Lists the capability and connection types available on an edge site, including each type's manifest. Use this to find the typeId and expected parameters for create_capability and create_connection.", + Description: "Lists the capability and connection types available on an edge site. Without query, returns just the ids (an edge site can have 80+ types, each with a large manifest). Pass query to filter to matching types and get their full manifest describing expected parameters for create_capability/create_connection.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, @@ -49,6 +51,43 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { return Output{}, err } - return Output{CapabilityTypes: capabilityTypes, ConnectionTypes: connectionTypes}, nil + query := strings.ToLower(strings.TrimSpace(input.Query)) + if query == "" { + return Output{ + CapabilityTypes: stripManifests(capabilityTypes), + ConnectionTypes: stripManifestsConn(connectionTypes), + }, nil + } + + var matchedCapabilities []clients.CapabilityType + for _, c := range capabilityTypes { + if strings.Contains(strings.ToLower(c.ID), query) { + matchedCapabilities = append(matchedCapabilities, c) + } + } + var matchedConnections []clients.ConnectionType + for _, c := range connectionTypes { + if strings.Contains(strings.ToLower(c.ID), query) { + matchedConnections = append(matchedConnections, c) + } + } + + return Output{CapabilityTypes: matchedCapabilities, ConnectionTypes: matchedConnections}, nil + } +} + +func stripManifests(types []clients.CapabilityType) []clients.CapabilityType { + out := make([]clients.CapabilityType, len(types)) + for i, t := range types { + out[i] = clients.CapabilityType{ID: t.ID} + } + return out +} + +func stripManifestsConn(types []clients.ConnectionType) []clients.ConnectionType { + out := make([]clients.ConnectionType, len(types)) + for i, t := range types { + out[i] = clients.ConnectionType{ID: t.ID} } + return out } diff --git a/pkg/tools/list_capability_types/tool_test.go b/pkg/tools/list_capability_types/tool_test.go index 18dc771..baab63f 100644 --- a/pkg/tools/list_capability_types/tool_test.go +++ b/pkg/tools/list_capability_types/tool_test.go @@ -38,6 +38,71 @@ func TestListCapabilityTypes(t *testing.T) { } } +func TestListCapabilityTypes_NoQueryStripsManifests(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/capabilityTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.CapabilityType) { + return http.StatusOK, []clients.CapabilityType{ + {ID: "jdbc-ingestion", Manifest: map[string]any{"huge": "manifest"}}, + {ID: "snowflake-synchronization", Manifest: map[string]any{"huge": "manifest"}}, + } + })) + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/connectionTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.ConnectionType) { + return http.StatusOK, []clients.ConnectionType{{ID: "Generic", Manifest: map[string]any{"huge": "manifest"}}} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: siteID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.CapabilityTypes) != 2 { + t.Fatalf("expected 2 capability types, got: %+v", output.CapabilityTypes) + } + for _, c := range output.CapabilityTypes { + if c.Manifest != nil { + t.Fatalf("expected manifest to be stripped without a query, got: %+v", c) + } + } +} + +func TestListCapabilityTypes_QueryFiltersAndKeepsManifest(t *testing.T) { + siteID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/capabilityTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.CapabilityType) { + return http.StatusOK, []clients.CapabilityType{ + {ID: "jdbc-ingestion", Manifest: map[string]any{"connection": "type"}}, + {ID: "snowflake-synchronization", Manifest: map[string]any{"other": "type"}}, + } + })) + handler.Handle("GET /edge/api/rest/v2/sites/"+siteID.String()+"/connectionTypes", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.ConnectionType) { + return http.StatusOK, []clients.ConnectionType{{ID: "Generic"}} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: siteID.String(), Query: "jdbc"}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if len(output.CapabilityTypes) != 1 || output.CapabilityTypes[0].ID != "jdbc-ingestion" { + t.Fatalf("expected only jdbc-ingestion to match, got: %+v", output.CapabilityTypes) + } + if output.CapabilityTypes[0].Manifest == nil { + t.Fatalf("expected manifest to be kept when query filters results") + } + if len(output.ConnectionTypes) != 0 { + t.Fatalf("expected no connection types to match 'jdbc', got: %+v", output.ConnectionTypes) + } +} + func TestListCapabilityTypes_InvalidEdgeSiteID(t *testing.T) { client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{EdgeSiteID: "not-a-uuid"}) diff --git a/pkg/tools/register.go b/pkg/tools/register.go index 0154e4d..52bb013 100644 --- a/pkg/tools/register.go +++ b/pkg/tools/register.go @@ -16,8 +16,12 @@ import ( "github.com/collibra/chip/pkg/tools/discover_business_glossary" "github.com/collibra/chip/pkg/tools/discover_data_assets" "github.com/collibra/chip/pkg/tools/edit_asset" + "github.com/collibra/chip/pkg/tools/find_connections" + "github.com/collibra/chip/pkg/tools/find_domain_types" + "github.com/collibra/chip/pkg/tools/find_users" "github.com/collibra/chip/pkg/tools/get_asset_details" "github.com/collibra/chip/pkg/tools/get_business_term_data" + "github.com/collibra/chip/pkg/tools/get_catalog_job_status" "github.com/collibra/chip/pkg/tools/get_column_semantics" "github.com/collibra/chip/pkg/tools/get_context_specification" "github.com/collibra/chip/pkg/tools/get_data_source_setup_guide" @@ -92,16 +96,24 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, list_context_specifications.NewTool(client)) toolRegister(server, toolConfig, get_context_specification.NewTool(client)) } + toolRegister(server, toolConfig, find_users.NewTool(client)) + toolRegister(server, toolConfig, find_domain_types.NewTool(client)) + toolRegister(server, toolConfig, find_connections.NewTool(client)) toolRegister(server, toolConfig, list_edge_sites.NewTool(client)) toolRegister(server, toolConfig, list_capability_types.NewTool(client)) toolRegister(server, toolConfig, create_community.NewTool(client)) toolRegister(server, toolConfig, create_domain.NewTool(client)) - toolRegister(server, toolConfig, upload_file.NewTool(client)) + if toolConfig.AllowLocalFileUpload { + toolRegister(server, toolConfig, upload_file.NewToolWithFilePath(client)) + } else { + toolRegister(server, toolConfig, upload_file.NewTool(client)) + } toolRegister(server, toolConfig, get_data_source_setup_guide.NewTool(client)) toolRegister(server, toolConfig, create_connection.NewTool(client)) toolRegister(server, toolConfig, create_capability.NewTool(client)) toolRegister(server, toolConfig, test_connection.NewTool(client)) toolRegister(server, toolConfig, get_job_status.NewTool(client)) + toolRegister(server, toolConfig, get_catalog_job_status.NewTool(client)) toolRegister(server, toolConfig, configure_database.NewTool(client)) toolRegister(server, toolConfig, start_ingestion.NewTool(client)) diff --git a/pkg/tools/start_ingestion/tool.go b/pkg/tools/start_ingestion/tool.go index 2156cc0..cab4a51 100644 --- a/pkg/tools/start_ingestion/tool.go +++ b/pkg/tools/start_ingestion/tool.go @@ -19,7 +19,7 @@ type Input struct { } type Output struct { - Job *clients.Job `json:"job,omitempty" jsonschema:"The started synchronization job."` + Job *clients.Job `json:"job,omitempty" jsonschema:"The started synchronization job. Poll its id with get_catalog_job_status (NOT get_job_status, which is for Edge-site jobs) to see when it completes."` Success bool `json:"success" jsonschema:"Whether the ingestion job was started."` Error string `json:"error,omitempty" jsonschema:"Error message if starting the job failed."` } @@ -28,7 +28,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "start_ingestion", Title: "Start Database Ingestion", - Description: "Triggers the jdbc-ingestion capability run for a registered Database asset, synchronizing its configured schemas/tables into the catalog. The database must already be registered and configured via configure_database.", + Description: "Triggers the jdbc-ingestion capability run for a registered Database asset, synchronizing its configured schemas/tables into the catalog. The database must already be registered and configured via configure_database. Poll the returned job's id with get_catalog_job_status to confirm it actually completed — a 202/success response here only means the job was accepted, not that ingestion finished.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/upload_file/tool.go b/pkg/tools/upload_file/tool.go index 5ed0971..db05c2a 100644 --- a/pkg/tools/upload_file/tool.go +++ b/pkg/tools/upload_file/tool.go @@ -1,25 +1,85 @@ -// Package upload_file implements the upload_file MCP tool: uploads an arbitrary file -// to an edge site and returns an artifact URI usable as the value of any FILE-type -// connection or capability parameter (JDBC drivers, TLS certs, private keys, keytabs, -// etc.) — not just the JDBC-driver convenience built into create_connection. +// Package upload_file implements the upload_file MCP tool: uploads a file to an +// edge site and returns an artifact URI usable as the value of any FILE-type +// connection or capability parameter (JDBC drivers, TLS certs, private keys, +// keytabs, etc.). +// +// Policy: this tool deliberately does NOT fetch file content from an arbitrary URL +// (e.g. Maven Central). For JDBC drivers specifically, Collibra only supports/ +// verifies drivers distributed through Collibra Marketplace, and Marketplace +// downloads require an authenticated account — there is no public, unauthenticated +// download endpoint to automate. get_data_source_setup_guide's marketplaceUrl field +// points to the correct listing; the user must download the driver from there +// themselves and hand the bytes to this tool. Do not substitute an unverified source +// (Maven Central, GitHub releases, etc.) for a driver jar just because it's more +// convenient to automate — ask the user for the file instead. +// +// Large files and the two Input variants (documented gap for chip-service): +// +// contentBase64 has a hard ceiling that isn't really about encoding or transport: any +// value passed as a tool-call argument has to be generated by the calling model +// itself, token by token, as part of its own output. A 96MB JDBC driver jar becomes +// ~130MB of base64 text the model would have to produce — that holds regardless of +// encoding, MCP transport, or how this tool is implemented. +// +// filePath (reading the file directly off local disk, server-side, bypassing the +// model entirely) fixes this — but only makes sense when chip is running as a local +// process with access to the caller's own filesystem. That's true for the standalone +// `chip` binary (a real, documented distribution channel — see the repo README — +// where a user runs chip locally against their own tenant), but not for chip-service, +// the product's remote, multi-tenant deployment: there, "local disk" means the server +// container's disk, not any user's machine, and a filePath input would be a genuine +// arbitrary-file-read vulnerability if ever reachable. +// +// Both binaries call the same tools.RegisterAll, so filePath support is exposed as a +// second, separate Input type (InputWithFilePath) registered under an unchanged tool +// name ("upload_file") only when chip.ServerToolConfig.AllowLocalFileUpload is true. +// This is gated at registration time, not inside the handler on every call: only +// cmd/chip's own startup wiring ever sets AllowLocalFileUpload, so chip-service's +// binary has no code path that could turn it on, even by mistake. If chip-service +// ever needs equivalent large-file support, it needs its own solution appropriate to +// a remote, multi-tenant server (e.g. a presigned upload step, or client-side file +// attachment support in whatever chat surface embeds it) — not this field. +// +// For chip-service today, the only large-file workaround is manual: guide the user +// to create the whole connection through the Collibra Edge UI (Settings > Edge > +// site > Connections > New), supplying the exact field values yourself (driver +// class, connection string — from get_data_source_setup_guide) so they only have to +// pick the file and click through, since there is no standalone "upload just the +// driver, get a reference back" screen or API — confirmed by checking the newer +// catalog-backed driver API (POST /rest/edge/v1/drivers/jars), which has no UI in +// front of it at all, and the frontend's only file-upload path is the legacy one +// embedded inside the connection-creation form's own FILE-type field. Once the user +// confirms the connection is saved, use find_connections (by name) to pick up its id +// and proceed with create_capability/configure_database/start_ingestion against it — +// skip create_connection entirely for that connection, since it already exists. package upload_file import ( "context" "encoding/base64" "fmt" - "io" "net/http" + "os" "github.com/collibra/chip/pkg/chip" "github.com/collibra/chip/pkg/clients" "github.com/modelcontextprotocol/go-sdk/mcp" ) +// Input is used when local file uploads are not enabled (the default, and always the +// case for chip-service) — content must be supplied inline as base64. type Input struct { - Filename string `json:"filename" jsonschema:"The filename to upload as (e.g. 'client-cert.pem'). Determines the file extension DGC uses to infer the artifact type."` - URL string `json:"url,omitempty" jsonschema:"A URL to download the file's content from (e.g. a Maven Central artifact URL or an internal file server). Exactly one of url or contentBase64 must be provided."` - ContentBase64 string `json:"contentBase64,omitempty" jsonschema:"The file's content, base64-encoded. Exactly one of url or contentBase64 must be provided."` + Filename string `json:"filename" jsonschema:"The filename to upload as (e.g. 'client-cert.pem', 'snowflake-jdbc-3.23.2.jar'). Determines the file extension DGC uses to infer the artifact type."` + ContentBase64 string `json:"contentBase64" jsonschema:"The file's content, base64-encoded. For a JDBC driver, this must come from a file the user downloaded from Collibra Marketplace (see get_data_source_setup_guide's marketplaceUrl) — never fetch a driver from an arbitrary URL like Maven Central; ask the user for the file instead. Large files (tens of MB, as most JDBC drivers are) will not fit here — if the content would be too large to include, ask the user to create the connection manually via the Collibra Edge UI instead and pick it up afterward with find_connections."` +} + +// InputWithFilePath is used when AllowLocalFileUpload is true (only ever set by the +// standalone chip binary, never chip-service) — adds filePath as an alternative to +// contentBase64 for large files. +type InputWithFilePath struct { + Filename string `json:"filename" jsonschema:"The filename to upload as (e.g. 'client-cert.pem', 'snowflake-jdbc-3.23.2.jar'). Determines the file extension DGC uses to infer the artifact type."` + ContentBase64 string `json:"contentBase64,omitempty" jsonschema:"The file's content, base64-encoded. Exactly one of contentBase64 or filePath must be provided. Prefer filePath for large files (tens of MB, as most JDBC drivers are) — contentBase64 will not fit them in a tool call."` + FilePath string `json:"filePath,omitempty" jsonschema:"Absolute path to a local file to read and upload directly from this machine's disk, without passing its content through the conversation. Available because this chip instance is running locally with access to your own filesystem. Exactly one of contentBase64 or filePath must be provided."` } type Output struct { @@ -28,68 +88,67 @@ type Output struct { Error string `json:"error,omitempty" jsonschema:"Error message if the upload failed."` } +// NewTool registers upload_file without filePath support (contentBase64 only). Used +// whenever AllowLocalFileUpload is false — always true for chip-service. func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "upload_file", Title: "Upload File to Edge Site", - Description: "Uploads a file (JDBC driver, TLS certificate, private key, keytab, etc.) to an edge site and returns an artifact URI usable as the value of any FILE-type connection or capability parameter.", - Handler: handler(collibraClient), + Description: "Uploads a file (JDBC driver, TLS certificate, private key, keytab, etc.) to an edge site and returns an artifact URI usable as the value of any FILE-type connection or capability parameter. Takes file content directly (base64) — does not fetch from a URL. Only suitable for small files (contentBase64 has to fit in a tool call); for a large file (e.g. most JDBC drivers), guide the user to create the connection manually via the Collibra Edge UI and pick it up with find_connections. For JDBC drivers, only use content downloaded from Collibra Marketplace (verified/supported), never an arbitrary host; get the Marketplace listing URL from get_data_source_setup_guide.", + Handler: func(ctx context.Context, input Input) (Output, error) { + return upload(ctx, collibraClient, input.Filename, input.ContentBase64, "") + }, Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, } } -func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { - return func(ctx context.Context, input Input) (Output, error) { - if input.Filename == "" { - return Output{}, fmt.Errorf("filename is required") - } - if (input.URL == "") == (input.ContentBase64 == "") { - return Output{}, fmt.Errorf("exactly one of url or contentBase64 must be provided") - } - - content, err := resolveContent(ctx, input) - if err != nil { - return Output{Success: false, Error: err.Error()}, nil - } - - uri, err := clients.UploadFile(ctx, collibraClient, input.Filename, content) - if err != nil { - return Output{Success: false, Error: fmt.Sprintf("failed to upload file: %s", err.Error())}, nil - } - - return Output{URI: uri, Success: true}, nil +// NewToolWithFilePath registers upload_file with filePath support added. Only called +// when chip.ServerToolConfig.AllowLocalFileUpload is true — see package doc comment +// for why this must never happen in chip-service. +func NewToolWithFilePath(collibraClient *http.Client) *chip.Tool[InputWithFilePath, Output] { + return &chip.Tool[InputWithFilePath, Output]{ + Name: "upload_file", + Title: "Upload File to Edge Site", + Description: "Uploads a file (JDBC driver, TLS certificate, private key, keytab, etc.) to an edge site and returns an artifact URI usable as the value of any FILE-type connection or capability parameter. Does not fetch from a URL. Prefer filePath for large files (e.g. most JDBC drivers) — it reads the file directly from this machine's disk, avoiding the base64-in-tool-call size limit. For JDBC drivers, only use a file downloaded from Collibra Marketplace (verified/supported), never an arbitrary host; get the Marketplace listing URL from get_data_source_setup_guide.", + Handler: func(ctx context.Context, input InputWithFilePath) (Output, error) { + return upload(ctx, collibraClient, input.Filename, input.ContentBase64, input.FilePath) + }, + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, } } -func resolveContent(ctx context.Context, input Input) ([]byte, error) { - if input.ContentBase64 != "" { - content, err := base64.StdEncoding.DecodeString(input.ContentBase64) - if err != nil { - return nil, fmt.Errorf("decoding contentBase64: %w", err) - } - return content, nil - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, input.URL, nil) - if err != nil { - return nil, fmt.Errorf("building download request: %w", err) +func upload(ctx context.Context, collibraClient *http.Client, filename, contentBase64, filePath string) (Output, error) { + if filename == "" { + return Output{}, fmt.Errorf("filename is required") } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("downloading file: %w", err) + hasContent := contentBase64 != "" + hasFilePath := filePath != "" + if hasContent == hasFilePath { + return Output{}, fmt.Errorf("exactly one of contentBase64 or filePath must be provided") } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("downloading file: unexpected status %d", resp.StatusCode) + var content []byte + if hasFilePath { + data, err := os.ReadFile(filePath) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to read file %q: %s", filePath, err.Error())}, nil + } + content = data + } else { + decoded, err := base64.StdEncoding.DecodeString(contentBase64) + if err != nil { + return Output{}, fmt.Errorf("decoding contentBase64: %w", err) + } + content = decoded } - content, err := io.ReadAll(resp.Body) + uri, err := clients.UploadFile(ctx, collibraClient, filename, content) if err != nil { - return nil, fmt.Errorf("reading downloaded content: %w", err) + return Output{Success: false, Error: fmt.Sprintf("failed to upload file: %s", err.Error())}, nil } - return content, nil + return Output{URI: uri, Success: true}, nil } diff --git a/pkg/tools/upload_file/tool_test.go b/pkg/tools/upload_file/tool_test.go index c127f00..fe4e8a9 100644 --- a/pkg/tools/upload_file/tool_test.go +++ b/pkg/tools/upload_file/tool_test.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "net/http" "net/http/httptest" + "os" "testing" "github.com/collibra/chip/pkg/tools/testutil" @@ -37,25 +38,49 @@ func TestUploadFile_FromBase64Content(t *testing.T) { } } -func TestUploadFile_FromURL(t *testing.T) { - edgeHandler := http.NewServeMux() - edgeHandler.HandleFunc("POST /edge/api/rest/v2/upload", func(w http.ResponseWriter, r *http.Request) { +func TestUploadFile_RequiresContentBase64(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Filename: "x"}) + if err == nil { + t.Fatalf("expected an error when contentBase64 is not provided") + } +} + +func TestUploadFile_RequiresFilename(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ContentBase64: "Zm9v"}) + if err == nil { + t.Fatalf("expected an error when filename is not provided") + } +} + +func TestUploadFile_WithFilePath_FromDisk(t *testing.T) { + tmpFile, err := os.CreateTemp(t.TempDir(), "driver-*.jar") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + if _, err := tmpFile.Write([]byte("fake jar bytes")); err != nil { + t.Fatalf("failed to write temp file: %v", err) + } + if err := tmpFile.Close(); err != nil { + t.Fatalf("failed to close temp file: %v", err) + } + + handler := http.NewServeMux() + handler.HandleFunc("POST /edge/api/rest/v2/upload", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("jar://test-uuid/driver.jar")) }) - edgeServer := httptest.NewServer(edgeHandler) - defer edgeServer.Close() - - sourceServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = w.Write([]byte("fake jar bytes")) - })) - defer sourceServer.Close() + server := httptest.NewServer(handler) + defer server.Close() - client := testutil.NewClient(edgeServer) - output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + client := testutil.NewClient(server) + output, err := tools.NewToolWithFilePath(client).Handler(t.Context(), tools.InputWithFilePath{ Filename: "driver.jar", - URL: sourceServer.URL, + FilePath: tmpFile.Name(), }) if err != nil { t.Fatalf("expected no error, got: %v", err) @@ -63,22 +88,40 @@ func TestUploadFile_FromURL(t *testing.T) { if !output.Success { t.Fatalf("expected success, got error: %s", output.Error) } + if output.URI != "jar://test-uuid/driver.jar" { + t.Fatalf("unexpected uri: %s", output.URI) + } } -func TestUploadFile_RequiresExactlyOneSource(t *testing.T) { +func TestUploadFile_WithFilePath_RequiresExactlyOneSource(t *testing.T) { client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) - _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{Filename: "x"}) + _, err := tools.NewToolWithFilePath(client).Handler(t.Context(), tools.InputWithFilePath{Filename: "x"}) if err == nil { - t.Fatalf("expected an error when neither url nor contentBase64 is provided") + t.Fatalf("expected an error when neither contentBase64 nor filePath is provided") } - _, err = tools.NewTool(client).Handler(t.Context(), tools.Input{ + _, err = tools.NewToolWithFilePath(client).Handler(t.Context(), tools.InputWithFilePath{ Filename: "x", - URL: "http://example.com/x", ContentBase64: "Zm9v", + FilePath: "/tmp/x", }) if err == nil { - t.Fatalf("expected an error when both url and contentBase64 are provided") + t.Fatalf("expected an error when both contentBase64 and filePath are provided") + } +} + +func TestUploadFile_WithFilePath_ReadError(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + + output, err := tools.NewToolWithFilePath(client).Handler(t.Context(), tools.InputWithFilePath{ + Filename: "driver.jar", + FilePath: "/nonexistent/path/driver.jar", + }) + if err != nil { + t.Fatalf("expected no error (failure reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure for a nonexistent file") } } From 82e9fffc392d78b58a3d972a597cefaacfdf94e3 Mon Sep 17 00:00:00 2001 From: Dary Schaeken Date: Mon, 6 Jul 2026 15:03:25 +0200 Subject: [PATCH 03/10] feat: add OAuth2 scopes to edge-management tools Sets Permissions on the tools that call edge-management's connection, capability, and file-upload endpoints so chip-service's token exchange narrows to dgc.edge-integration-capability-manage (writes) or dgc.edge-view-connections-and-capabilities (reads), traced against EdgeAuthorizationHelper and AccessTokenGrantedAuthoritiesConverterTest. Catalog/DGC-core tools are left unscoped pending verification of that separate authorization surface. --- pkg/tools/create_capability/tool.go | 2 +- pkg/tools/create_connection/tool.go | 2 +- pkg/tools/find_connections/tool.go | 2 +- pkg/tools/list_capability_types/tool.go | 2 +- pkg/tools/list_edge_sites/tool.go | 2 +- pkg/tools/test_connection/tool.go | 2 +- pkg/tools/upload_file/tool.go | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/tools/create_capability/tool.go b/pkg/tools/create_capability/tool.go index 90d7ba2..f64c781 100644 --- a/pkg/tools/create_capability/tool.go +++ b/pkg/tools/create_capability/tool.go @@ -35,7 +35,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { Title: "Create or Update Edge Capability", Description: "Creates or updates an Edge capability (e.g. jdbc-ingestion) via the private Edge capability management API. Does not run the capability — use start_ingestion to trigger a jdbc-ingestion run.", Handler: handler(collibraClient), - Permissions: []string{}, + Permissions: []string{"dgc.edge-integration-capability-manage"}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, } } diff --git a/pkg/tools/create_connection/tool.go b/pkg/tools/create_connection/tool.go index eec432f..f595afe 100644 --- a/pkg/tools/create_connection/tool.go +++ b/pkg/tools/create_connection/tool.go @@ -51,7 +51,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { Title: "Create or Update Edge Connection", Description: "Creates or updates an Edge connection (e.g. a JDBC connection for the jdbc-ingestion capability) via the private Edge connection management API. Does not download driver files itself — see upload_file for how driver jars must be sourced (Collibra Marketplace only, not arbitrary URLs).", Handler: handler(collibraClient), - Permissions: []string{}, + Permissions: []string{"dgc.edge-integration-capability-manage"}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, } } diff --git a/pkg/tools/find_connections/tool.go b/pkg/tools/find_connections/tool.go index 74e336e..605ad77 100644 --- a/pkg/tools/find_connections/tool.go +++ b/pkg/tools/find_connections/tool.go @@ -31,7 +31,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { Title: "Find Edge Connections", Description: "Finds Edge connections by name (and optionally edge site). Use this to pick up a connection that already exists — e.g. one the user created manually via the DGC/Edge UI for a driver file too large to pass through upload_file — instead of creating a duplicate with create_connection.", Handler: handler(collibraClient), - Permissions: []string{}, + Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, } } diff --git a/pkg/tools/list_capability_types/tool.go b/pkg/tools/list_capability_types/tool.go index 016c3ac..4717f4f 100644 --- a/pkg/tools/list_capability_types/tool.go +++ b/pkg/tools/list_capability_types/tool.go @@ -30,7 +30,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { Title: "List Capability and Connection Types", Description: "Lists the capability and connection types available on an edge site. Without query, returns just the ids (an edge site can have 80+ types, each with a large manifest). Pass query to filter to matching types and get their full manifest describing expected parameters for create_capability/create_connection.", Handler: handler(collibraClient), - Permissions: []string{}, + Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, } } diff --git a/pkg/tools/list_edge_sites/tool.go b/pkg/tools/list_edge_sites/tool.go index e9c87ea..d8cd8d6 100644 --- a/pkg/tools/list_edge_sites/tool.go +++ b/pkg/tools/list_edge_sites/tool.go @@ -24,7 +24,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { Title: "List Edge Sites", Description: "Lists available Edge sites, including their id, name, and status. Use this to find the edgeSiteId needed by create_connection and create_capability.", Handler: handler(collibraClient), - Permissions: []string{}, + Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, } } diff --git a/pkg/tools/test_connection/tool.go b/pkg/tools/test_connection/tool.go index 49cf5ee..bbf70bf 100644 --- a/pkg/tools/test_connection/tool.go +++ b/pkg/tools/test_connection/tool.go @@ -31,7 +31,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { Title: "Test Edge Connection", Description: "Tests whether an Edge connection can reach its data source. Without timeoutSec, returns immediately with a jobId to poll via get_job_status; with timeoutSec, waits up to that many seconds and returns the final result.", Handler: handler(collibraClient), - Permissions: []string{}, + Permissions: []string{"dgc.edge-integration-capability-manage"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, } } diff --git a/pkg/tools/upload_file/tool.go b/pkg/tools/upload_file/tool.go index db05c2a..50c711d 100644 --- a/pkg/tools/upload_file/tool.go +++ b/pkg/tools/upload_file/tool.go @@ -98,7 +98,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { Handler: func(ctx context.Context, input Input) (Output, error) { return upload(ctx, collibraClient, input.Filename, input.ContentBase64, "") }, - Permissions: []string{}, + Permissions: []string{"dgc.edge-integration-capability-manage"}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, } } @@ -114,7 +114,7 @@ func NewToolWithFilePath(collibraClient *http.Client) *chip.Tool[InputWithFilePa Handler: func(ctx context.Context, input InputWithFilePath) (Output, error) { return upload(ctx, collibraClient, input.Filename, input.ContentBase64, input.FilePath) }, - Permissions: []string{}, + Permissions: []string{"dgc.edge-integration-capability-manage"}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, } } From 6e1d5f74bc4a9f3302fd9892d77fb15f2f8369d4 Mon Sep 17 00:00:00 2001 From: Dary Schaeken Date: Tue, 7 Jul 2026 08:46:24 +0200 Subject: [PATCH 04/10] docs: add jdbc-ingestion skill and document tool set in README/SKILLS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a collibra/jdbc-ingestion skill covering the full connect → register → run sequence, the two separate job-id spaces, the Marketplace-only driver policy, and the caution against using a raw run-capability tool for jdbc-ingestion instead of start_ingestion. Cross-links it from the discovery and index skills, and backfills README's tool list, which never listed this tool family. --- README.md | 18 +++- SKILLS.md | 1 + pkg/skills/files/collibra/discovery/SKILL.md | 2 +- pkg/skills/files/collibra/index/SKILL.md | 3 +- .../files/collibra/jdbc-ingestion/SKILL.md | 100 ++++++++++++++++++ 5 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 pkg/skills/files/collibra/jdbc-ingestion/SKILL.md diff --git a/README.md b/README.md index ba6d0df..deaefe7 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,15 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`discover_business_glossary`](pkg/tools/discover_business_glossary/) - Ask questions about terms and definitions. Note that this tool leverages Collibra AI and therefore consumes Collibra Units (CUs). **Requires:** `dgc.ai-copilot` - [`discover_data_assets`](pkg/tools/discover_data_assets/) - Query available data assets using natural language. Note that this tool leverages Collibra AI and therefore consumes Collibra Units (CUs). **Requires:** `dgc.ai-copilot` +- [`find_connections`](pkg/tools/find_connections/) - Find existing Edge connections by name, to reuse instead of duplicating. **Requires:** `dgc.edge-view-connections-and-capabilities` +- [`find_domain_types`](pkg/tools/find_domain_types/) - Resolve a domain type name (e.g. "Physical Data Dictionary") to its UUID +- [`find_users`](pkg/tools/find_users/) - Resolve a user name to their UUID, e.g. for a database asset's owners - [`get_asset_details`](pkg/tools/get_asset_details/) - Retrieve detailed information about specific assets by UUID, including the asset's assignable attribute schema (every attribute it can hold, including empty ones) - [`get_business_term_data`](pkg/tools/get_business_term_data/) - Trace a business term back to its connected physical data assets +- [`get_catalog_job_status`](pkg/tools/get_catalog_job_status/) - Poll the status of a DGC catalog job, e.g. a jdbc-ingestion sync started by `start_ingestion` - [`get_column_semantics`](pkg/tools/get_column_semantics/) - Retrieve data attributes, measures, and business assets connected to a column +- [`get_data_source_setup_guide`](pkg/tools/get_data_source_setup_guide/) - Look up JDBC driver class, connection string format, and connection properties for a data source from Collibra's documentation +- [`get_job_status`](pkg/tools/get_job_status/) - Poll the status of an Edge job, e.g. a connection test or capability run - [`get_lineage_downstream`](pkg/tools/get_lineage_downstream/) - Get downstream technical lineage (consumers) for a data entity - [`get_lineage_entity`](pkg/tools/get_lineage_entity/) - Get metadata about a specific entity in the technical lineage graph - [`get_lineage_transformation`](pkg/tools/get_lineage_transformation/) - Get details and logic of a specific data transformation @@ -22,7 +28,9 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`get_measure_data`](pkg/tools/get_measure_data/) - Trace a measure back to its underlying physical columns and tables - [`get_table_semantics`](pkg/tools/get_table_semantics/) - Retrieve the semantic layer for a table: columns, data attributes, and connected measures - [`list_asset_types`](pkg/tools/list_asset_types/) - List available asset types +- [`list_capability_types`](pkg/tools/list_capability_types/) - List capability and connection types available on an Edge site, with their manifests. **Requires:** `dgc.edge-view-connections-and-capabilities` - [`list_data_contract`](pkg/tools/list_data_contracts/) - List data contracts with pagination +- [`list_edge_sites`](pkg/tools/list_edge_sites/) - List available Edge sites. **Requires:** `dgc.edge-view-connections-and-capabilities` - [`prepare_create_asset`](pkg/tools/prepare_create_asset/) - Read-only companion to `create_asset`: enumerate available asset types and domains, resolve a UUID/publicId/displayName for either, and hydrate the scoped attribute and relation schema for a chosen pair - [`pull_data_contract_manifest`](pkg/tools/pull_data_contract_manifest/) - Download manifest for a data contract - [`search_asset_keyword`](pkg/tools/search_asset_keyword/) - Wildcard keyword search for assets; filters (status, community, domain, domain type, asset type, created-by) accept names or UUIDs @@ -30,11 +38,17 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`search_data_classification_match`](pkg/tools/search_data_classification_matches/) - Search for associations between data classes and assets. **Requires:** `dgc.classify`, `dgc.catalog` - [`search_lineage_entities`](pkg/tools/search_lineage_entities/) - Search for entities in the technical lineage graph - [`search_lineage_transformations`](pkg/tools/search_lineage_transformations/) - Search for transformations in the technical lineage graph +- [`test_connection`](pkg/tools/test_connection/) - Test whether an Edge connection can reach its data source. **Requires:** `dgc.edge-integration-capability-manage` ### Write Tools - [`add_data_classification_match`](pkg/tools/add_data_classification_match/) - Associate a data class with an asset. **Requires:** `dgc.classify`, `dgc.catalog` +- [`configure_database`](pkg/tools/configure_database/) - Register a database and configure its schema synchronization rules for jdbc-ingestion - [`create_asset`](pkg/tools/create_asset/) - Create a new asset of any type. Resolves `assetType` (UUID, publicId, or display name), `domain` (UUID or name), `status` (UUID or name), and attributes (by name or typeId) server-side; converts Markdown to HTML for `RICH_TEXT` attributes; gates on duplicate-name (default `allowDuplicate: false`) +- [`create_capability`](pkg/tools/create_capability/) - Create or update an Edge capability (e.g. jdbc-ingestion). **Requires:** `dgc.edge-integration-capability-manage` +- [`create_community`](pkg/tools/create_community/) - Create a DGC community +- [`create_connection`](pkg/tools/create_connection/) - Create or update an Edge connection (e.g. a JDBC connection). **Requires:** `dgc.edge-integration-capability-manage` +- [`create_domain`](pkg/tools/create_domain/) - Create a DGC domain within a community - [`edit_asset`](pkg/tools/edit_asset/) - Edit an existing asset via a list of typed operations: - `set_attribute`, `add_attribute`, `remove_attribute` - set an attribute value (creates if empty, updates if present), append an extra value to a multi-valued attribute, or clear one (e.g. `Definition`, `Note`) - `update_property` - rename the asset (`name`), change its `displayName`, or change its `statusId` (status name or UUID accepted) @@ -45,6 +59,8 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`init_data_contract`](pkg/tools/init_data_contract/) - Initialize a new data contract asset governing a Data Product Port, with an optional initial manifest. **Requires:** `dgc.data-contract` - [`push_data_contract_manifest`](pkg/tools/push_data_contract_manifest/) - Upload manifest for a data contract. **Requires:** `dgc.data-contract` - [`remove_data_classification_match`](pkg/tools/remove_data_classification_match/) - Remove a classification match. **Requires:** `dgc.classify`, `dgc.catalog`, `dgc.data-classes-edit` +- [`start_ingestion`](pkg/tools/start_ingestion/) - Trigger a jdbc-ingestion capability run for a registered database +- [`upload_file`](pkg/tools/upload_file/) - Upload a file (JDBC driver, TLS certificate, private key, keytab, …) to an Edge site. **Requires:** `dgc.edge-integration-capability-manage` ## Quick Start @@ -213,7 +229,7 @@ Some functionality ships behind an opt-in `experimental` flag. These features ar - `context-specifications` — Context specification tools: `list_context_specifications`, `get_context_specification`, and the `contextSpecificationId` parameter on `get_asset_details`. These tools generate structured YAML context for assets using the Semantic Blueprint API. -- `skills` — Embedded skill catalog served via two additional tools, `list_collibra_skills` and `load_collibra_skill`. Skills are short Markdown guides that document multi-step Collibra workflows (discovery, lineage, asset create/edit, …) for the connecting LLM. See [SKILLS.md](SKILLS.md) for the catalog. +- `skills` — Embedded skill catalog served via two additional tools, `list_collibra_skills` and `load_collibra_skill`. Skills are short Markdown guides that document multi-step Collibra workflows (discovery, lineage, asset create/edit, jdbc-ingestion setup, …) for the connecting LLM. See [SKILLS.md](SKILLS.md) for the catalog. Point chip at an **external skills directory** with `--skills-dir=` (or `COLLIBRA_MCP_SKILLS_DIR`, or `mcp.skills-dir` in YAML) to add your own skills on top of the embedded ones. The expected layout is `///SKILL.md` (with optional `references/*.md` and `_shared/*.md` siblings) — same as the bundled catalog. External skills whose name matches an embedded skill (e.g. `collibra/lineage`) **fully replace** the embedded entry, including its resources, so you can override the shipped guides without rebuilding chip. `~` and `~user` in the path are expanded. diff --git a/SKILLS.md b/SKILLS.md index 2c9e872..fa08444 100644 --- a/SKILLS.md +++ b/SKILLS.md @@ -17,6 +17,7 @@ those tools. Skill content lives in [`pkg/skills/files/collibra/`](pkg/skills/fi | `collibra/lineage` | Technical lineage; DGC UUID ↔ lineage entity ID bridge; column-level workaround | | `collibra/asset-create` | `create_asset` workflow; RICH_TEXT Markdown handling; duplicate gating | | `collibra/asset-edit` | `edit_asset` operation types | +| `collibra/jdbc-ingestion` | End-to-end jdbc-ingestion setup: Edge connection/capability, database registration, running and polling a sync | Each skill is one `SKILL.md` per directory, with frontmatter (`description`, `related`) and an optional `references/` directory for bundled reference documents. diff --git a/pkg/skills/files/collibra/discovery/SKILL.md b/pkg/skills/files/collibra/discovery/SKILL.md index ec5eb87..8a91067 100644 --- a/pkg/skills/files/collibra/discovery/SKILL.md +++ b/pkg/skills/files/collibra/discovery/SKILL.md @@ -1,6 +1,6 @@ --- description: Pick between natural-language semantic search and keyword/filter search to find Collibra assets and terms. -related: collibra/lineage, collibra/asset-create +related: collibra/lineage, collibra/asset-create, collibra/jdbc-ingestion --- # Discovery — finding assets and business terms diff --git a/pkg/skills/files/collibra/index/SKILL.md b/pkg/skills/files/collibra/index/SKILL.md index 46337e8..a2f9b15 100644 --- a/pkg/skills/files/collibra/index/SKILL.md +++ b/pkg/skills/files/collibra/index/SKILL.md @@ -1,6 +1,6 @@ --- description: Navigator for chip's Collibra skills. Start here when unsure which skill applies. -related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context +related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/jdbc-ingestion --- # Collibra skills — navigator @@ -19,6 +19,7 @@ must be bridged to another, and which permissions are required. | Modify an existing asset's attributes, relations, tags, status, or owners | `collibra/asset-edit` | | Register a table (and its dimension tables) as a Collibra Data Product with ports | `collibra/data-product-create` | | Generate governed YAML context (semantic blueprints, metric definitions) for an asset | `collibra/context` | +| Set up an Edge JDBC connection/capability and run jdbc-ingestion | `collibra/jdbc-ingestion` | If a task is a single tool call with no chaining (e.g. `get_asset_details` by UUID, `list_asset_types`, `pull_data_contract_manifest`), no skill is needed — the tool's own diff --git a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md new file mode 100644 index 0000000..ce8645f --- /dev/null +++ b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md @@ -0,0 +1,100 @@ +--- +description: Set up an Edge JDBC connection and capability, register a database, and run jdbc-ingestion end to end. +related: collibra/discovery +--- + +# JDBC ingestion — connect, register, and run + +Setting up jdbc-ingestion is a fixed sequence across two separate systems: Edge +(connection + capability) and DGC catalog (database registration + the actual sync). +No single tool does the whole thing — skipping a step or calling tools out of order is +the most common way this goes wrong. + +## The full sequence + +1. **`list_edge_sites`** — get the `edgeSiteId` you'll use everywhere below. +2. **`get_data_source_setup_guide`** (pass the data source name, e.g. `"Snowflake"`) — + before guessing any connection parameter. Returns the driver class, connection + string format, the Marketplace URL for the driver jar, and per-auth-method + properties. Treat a failed/empty lookup as "ask the user for the connection + properties directly," not as a bug to retry. +3. **`upload_file`** — upload the JDBC driver jar. **Driver files must come from + Collibra Marketplace only** (the URL from step 2) — never fetch one from Maven + Central or another host, and never invent one. If the file is too large for + `contentBase64` (most JDBC drivers are tens of MB) and this chip instance doesn't + have `filePath` enabled, stop and guide the user to create the connection manually + in the Edge UI instead — see "Large driver files" below. +4. **`create_connection`** — `typeId: "Generic"` (not a vendor-specific connection + type — jdbc-ingestion requires the Generic JDBC connection type). Fixed manifest + parameters (`driver-class`, `connection-string`, `driver-jar` as the `upload_file` + URI) go in `parameters`; open-ended vendor properties (Snowflake's Role, Warehouse, + User, private key file, etc.) go in `additionalProperties`, not `parameters`. +5. **`test_connection`** — verify it actually connects before building on top of it. + Pass `timeoutSec` for a synchronous result, or poll the returned `jobId` with + `get_job_status`. +6. **`create_capability`** — `typeId: "jdbc-ingestion"`, `parameters.connection` set to + the connection id from step 4. +7. Catalog scaffolding, only if it doesn't already exist: **`create_community`** → + **`find_domain_types`** (look up e.g. "Physical Data Dictionary") → + **`create_domain`**. **`find_users`** to resolve owner names to UUIDs for the next + step. +8. **`configure_database`** — one call does the full refresh-connection → + register-database → refresh-schema → set-sync-rules flow. Needs `communityId`, + `parentSystemId` (an existing System asset), and `ownerIds` (from step 7). If it + returns a retryable-looking error, it usually means the underlying async refresh + hasn't completed yet (cold-start data source connections can take 30–60s) — just + call it again; it's idempotent. +9. **`start_ingestion`** — triggers the actual sync using the database id from step 8. +10. **`get_catalog_job_status`** — poll the job id `start_ingestion` returned. A + 202/success from `start_ingestion` only means the job was accepted, not that + ingestion finished — always poll to confirm completion. + +## Two job-id spaces — do not cross them + +- **Edge-site jobs** (from `test_connection`) → poll with **`get_job_status`**. +- **DGC catalog jobs** (from `start_ingestion`) → poll with **`get_catalog_job_status`**. + +Passing one tool's job id to the other will fail or silently poll the wrong resource. +`start_ingestion`'s own `Job.id` is a catalog job id — never poll it with +`get_job_status`. + +## Do not use a raw "run capability" tool for this + +If this chip instance also exposes a generic "run capability" tool (e.g. +`edge_run_capability`, which wraps Edge's `POST /capabilities/{id}/run` directly): +**never use it to trigger a jdbc-ingestion run.** Always use `start_ingestion` instead. +`start_ingestion` goes through DGC's `synchronizeMetadata` endpoint, which does the +catalog database-registration bookkeeping (linking the job back to the Database asset, +updating sync state) that a raw capability run skips entirely — a direct run would +appear to succeed on the Edge side while leaving the catalog side out of sync. + +## Picking up an existing connection instead of duplicating one + +If a connection with the driver already exists (most commonly because the driver file +was too large to upload and the user created it manually via the Edge UI), use +**`find_connections`** (by name, optionally scoped to `edgeSiteId`) to get its id and +skip straight to step 6 — do not call `create_connection` again for it. + +## Large driver files + +`upload_file` takes file content as base64 in a tool call, which has a hard ceiling +that isn't about encoding — the content has to be generated by the calling model +itself, so a large jar (tens of MB) simply won't fit. Some chip instances (the +standalone `chip` binary, not chip-service) also accept a `filePath` input to read +the file directly off local disk instead. If neither `filePath` nor a small-enough +file is available: guide the user to create the whole connection manually via the Edge +UI (Settings > Edge > site > Connections > New), giving them the exact field values +(driver class, connection string — from `get_data_source_setup_guide`) so they only +need to pick the file and click through. Once they confirm it's saved, use +`find_connections` to pick it up and continue from step 6. + +## Hard rules + +1. **Never source a JDBC driver from anywhere but Collibra Marketplace.** Not Maven + Central, not a GitHub release, not a URL the user pastes that isn't the Marketplace + listing. If in doubt, ask the user to download the jar from the URL + `get_data_source_setup_guide` returns and hand you the file. +2. **`create_connection`'s `typeId` for jdbc-ingestion is `"Generic"`,** never a + vendor-specific connection type — jdbc-ingestion cannot use those. +3. **Always poll after `start_ingestion`.** A successful call only means the job was + accepted. From c5a552fbf0b53eb24975977764288c779b4ea016 Mon Sep 17 00:00:00 2001 From: Dary Schaeken Date: Tue, 7 Jul 2026 14:02:15 +0200 Subject: [PATCH 05/10] fix: require explicit database/schema/table scope in configure_database Adds a schemaNames guard mirroring the existing databaseName ambiguity check, and drops the silent include="*" default, so a multi-schema database can no longer be synchronized wholesale by omission. Updates the jdbc-ingestion skill to instruct agents to confirm scope with the user before calling, rather than defaulting to everything. --- .../files/collibra/jdbc-ingestion/SKILL.md | 39 ++++- pkg/tools/configure_database/tool.go | 71 ++++++++- pkg/tools/configure_database/tool_test.go | 143 ++++++++++++++++++ 3 files changed, 242 insertions(+), 11 deletions(-) diff --git a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md index ce8645f..942a63a 100644 --- a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md +++ b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md @@ -40,10 +40,13 @@ the most common way this goes wrong. step. 8. **`configure_database`** — one call does the full refresh-connection → register-database → refresh-schema → set-sync-rules flow. Needs `communityId`, - `parentSystemId` (an existing System asset), and `ownerIds` (from step 7). If it - returns a retryable-looking error, it usually means the underlying async refresh - hasn't completed yet (cold-start data source connections can take 30–60s) — just - call it again; it's idempotent. + `parentSystemId` (an existing System asset), and `ownerIds` (from step 7). **Never + default to "sync everything" — always confirm which database, which schemas, and + which tables with the user first.** See "Confirm scope before configuring the + database" below; the tool itself refuses to guess when there's more than one + database or schema. If it returns a retryable-looking error, it usually means the + underlying async refresh hasn't completed yet (cold-start data source connections + can take 30–60s) — just call it again; it's idempotent. 9. **`start_ingestion`** — triggers the actual sync using the database id from step 8. 10. **`get_catalog_job_status`** — poll the job id `start_ingestion` returned. A 202/success from `start_ingestion` only means the job was accepted, not that @@ -68,6 +71,31 @@ catalog database-registration bookkeeping (linking the job back to the Database updating sync state) that a raw capability run skips entirely — a direct run would appear to succeed on the Edge side while leaving the catalog side out of sync. +## Confirm scope before configuring the database + +`configure_database` will not silently register the wrong database or sync every table +in every schema — it's built to refuse ambiguity rather than guess: + +- **`databaseName`** is required if the data source exposes more than one + database/catalog through the connection. The tool errors and names the discovered + candidates if you omit it while more than one exists. +- **`schemaNames`** is required if the database has more than one schema. Same + behavior: omit it with multiple schemas discovered and the tool errors, listing them. + Pass the literal `["*"]` to configure every schema — only do this after the user has + actually said they want everything, not as a shortcut when you don't know yet. +- **`include`** has no default. You must always pass a table pattern (`"*"` for all + tables, or a specific comma-separated pattern) — there is no silent "sync + everything" fallback. + +The practical flow: call `configure_database` once with just the required identifiers +and let it fail on ambiguity to discover the candidate databases/schemas, **or** if you +already know multiple exist, ask the user up front which database, which schemas, and +which tables (or "all") they want before calling at all. Either way, do not pass `"*"` +for `schemaNames` or `include` on your own initiative — confirm with the user first. +The one exception: if discovery finds exactly one database and one schema, the tool +auto-selects them — no ambiguity to resolve, so no need to ask about *which* database +or schema, though you should still confirm the table pattern (`include`/`exclude`). + ## Picking up an existing connection instead of duplicating one If a connection with the driver already exists (most commonly because the driver file @@ -98,3 +126,6 @@ need to pick the file and click through. Once they confirm it's saved, use vendor-specific connection type — jdbc-ingestion cannot use those. 3. **Always poll after `start_ingestion`.** A successful call only means the job was accepted. +4. **Never pick a database, schema set, or table scope for the user.** When more than + one is possible, ask; when `configure_database` reports ambiguity, relay the + candidates and ask rather than picking one yourself or retrying with `"*"`. diff --git a/pkg/tools/configure_database/tool.go b/pkg/tools/configure_database/tool.go index fdf5495..106cc01 100644 --- a/pkg/tools/configure_database/tool.go +++ b/pkg/tools/configure_database/tool.go @@ -8,12 +8,20 @@ // with a bounded number of attempts; if a stage hasn't completed within that budget, // it returns an error asking the agent to retry the tool call (the refresh calls are // idempotent to re-issue). +// +// Ambiguity guards: databaseName is required when more than one database/catalog is +// discovered, schemaNames is required when more than one schema is discovered, and +// include has no default — all three force an explicit choice instead of silently +// registering the wrong database or syncing every schema/table. schemaNames/include +// still accept an explicit "*" to opt into "everything", but that must be a deliberate +// choice (confirmed with the user), never this tool's default behavior. package configure_database import ( "context" "fmt" "net/http" + "strings" "time" "github.com/collibra/chip/pkg/chip" @@ -41,7 +49,9 @@ type Input struct { OwnerIDs []string `json:"ownerIds" jsonschema:"UUIDs of the users to assign as owners of the Database asset. Use find_users to resolve a name (e.g. 'Admin') to its UUID."` Description string `json:"description,omitempty" jsonschema:"Optional description of the Database asset."` - Include string `json:"include,omitempty" jsonschema:"Optional. Comma-separated table name pattern to synchronize, '*' wildcard supported. Defaults to '*' (all tables)."` + SchemaNames []string `json:"schemaNames,omitempty" jsonschema:"Exact names of the schemas to configure for synchronization, as they appear at the data source. Required if the database exposes more than one schema; if there is exactly one, it is selected automatically. Pass the single value '*' to configure every discovered schema — only do this after confirming with the user which schemas they actually want synced, never as a silent default when multiple schemas exist."` + + Include string `json:"include" jsonschema:"Comma-separated table name pattern to synchronize, '*' wildcard supported. Required — there is no default. Confirm with the user which tables they want before calling; do not pass '*' (all tables) without asking first. Applies to every schema selected via schemaNames."` Exclude string `json:"exclude,omitempty" jsonschema:"Optional. Comma-separated table name pattern to exclude from synchronization."` TargetDomainID string `json:"targetDomainId,omitempty" jsonschema:"Optional. UUID of a domain to create synchronized assets in. If omitted, an automatically created domain per schema is used."` SkipViews bool `json:"skipViews,omitempty" jsonschema:"Optional. If true, database views are excluded from synchronization."` @@ -61,7 +71,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "configure_database", Title: "Configure Database for Ingestion", - Description: "Discovers a database through an Edge connection, registers it as a Database asset, and configures which tables get synchronized. Prerequisite for start_ingestion. Assumes the target community and parent System asset already exist (create_community/create_domain/create_asset), and that a jdbc-ingestion capability referencing this connection has already been created via create_capability — the discovery/refresh steps here only find data because that capability actually performs the crawl; without one, this fails with a discovery-timeout-shaped error even though the real cause is the missing capability.", + Description: "Discovers a database through an Edge connection, registers it as a Database asset, and configures which schemas/tables get synchronized. Prerequisite for start_ingestion. Assumes the target community and parent System asset already exist (create_community/create_domain/create_asset), and that a jdbc-ingestion capability referencing this connection has already been created via create_capability — the discovery/refresh steps here only find data because that capability actually performs the crawl; without one, this fails with a discovery-timeout-shaped error even though the real cause is the missing capability. If more than one database or schema is discovered, or include isn't provided, this returns an error naming the candidates instead of guessing — confirm the database, schemas, and table pattern with the user, do not default to configuring everything.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, @@ -88,6 +98,9 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { if err := validation.UUIDOptional("targetDomainId", input.TargetDomainID); err != nil { return Output{}, err } + if strings.TrimSpace(input.Include) == "" { + return Output{}, fmt.Errorf("include is required — confirm with the user which tables to synchronize (or that they want '*', all tables) before calling") + } databaseConnection, err := discoverDatabaseConnection(ctx, collibraClient, input.EdgeConnectionID, input.DatabaseName) if err != nil { @@ -105,14 +118,14 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { return Output{Success: false, Error: fmt.Sprintf("failed to register database: %s", err.Error())}, nil } - schemaConnections, err := discoverSchemaConnections(ctx, collibraClient, databaseConnection.ID) + discoveredSchemaConnections, err := discoverSchemaConnections(ctx, collibraClient, databaseConnection.ID) if err != nil { return Output{Database: database, Success: false, Error: err.Error()}, nil } - include := input.Include - if include == "" { - include = "*" + schemaConnections, err := selectSchemaConnections(discoveredSchemaConnections, input.SchemaNames) + if err != nil { + return Output{Database: database, Success: false, Error: err.Error()}, nil } configurations := make([]clients.SchemaMetadataConfiguration, len(schemaConnections)) @@ -121,7 +134,7 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { SchemaConnectionID: schemaConnection.ID, SynchronizationRules: []clients.MetadataSynchronizationRule{ { - Include: include, + Include: input.Include, Exclude: input.Exclude, TargetDomainID: input.TargetDomainID, SkipViews: input.SkipViews, @@ -207,3 +220,47 @@ func discoverSchemaConnections(ctx context.Context, client *http.Client, databas return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or no capability referencing this connection exists yet — verify with create_capability", databaseConnectionID, pollAttempts) } + +// selectSchemaConnections narrows discovered to the schemas named in schemaNames. An +// empty schemaNames is only accepted when there's no ambiguity (a single discovered +// schema); with more than one discovered schema, the caller must either name the ones +// they want or pass ["*"] to explicitly opt into configuring all of them — this +// mirrors discoverDatabaseConnection's databaseName requirement, so a multi-schema +// database can never be synchronized wholesale by omission. +func selectSchemaConnections(discovered []clients.SchemaConnection, schemaNames []string) ([]clients.SchemaConnection, error) { + if len(schemaNames) == 1 && schemaNames[0] == "*" { + return discovered, nil + } + + if len(schemaNames) == 0 { + if len(discovered) > 1 { + names := make([]string, len(discovered)) + for i, s := range discovered { + names[i] = s.Name + } + return nil, fmt.Errorf("multiple schemas discovered (%s); specify schemaNames to select which to configure, or pass [\"*\"] to configure all of them", strings.Join(names, ", ")) + } + return discovered, nil + } + + byName := make(map[string]clients.SchemaConnection, len(discovered)) + for _, s := range discovered { + byName[s.Name] = s + } + + selected := make([]clients.SchemaConnection, 0, len(schemaNames)) + var missing []string + for _, name := range schemaNames { + s, ok := byName[name] + if !ok { + missing = append(missing, name) + continue + } + selected = append(selected, s) + } + if len(missing) > 0 { + return nil, fmt.Errorf("schema(s) not found among discovered schemas: %s", strings.Join(missing, ", ")) + } + + return selected, nil +} diff --git a/pkg/tools/configure_database/tool_test.go b/pkg/tools/configure_database/tool_test.go index 9d57f17..9234604 100644 --- a/pkg/tools/configure_database/tool_test.go +++ b/pkg/tools/configure_database/tool_test.go @@ -84,6 +84,7 @@ func TestConfigureDatabase_Success(t *testing.T) { CommunityID: communityID.String(), ParentSystemID: systemID.String(), OwnerIDs: []string{ownerID.String()}, + Include: "*", }) if err != nil { t.Fatalf("expected no error, got: %v", err) @@ -99,6 +100,147 @@ func TestConfigureDatabase_Success(t *testing.T) { } } +func TestConfigureDatabase_MissingInclude(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err == nil { + t.Fatalf("expected an error when include is omitted") + } +} + +func TestConfigureDatabase_MultipleSchemasRequireSchemaNames(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + dbConnID, _ := uuid.NewUUID() + databaseID, _ := uuid.NewUUID() + publicSchemaID, _ := uuid.NewUUID() + privateSchemaID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.DatabaseConnection{ + {ID: dbConnID.String(), Name: "source", EdgeConnectionID: edgeConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { + return http.StatusCreated, clients.Database{ID: databaseID.String(), Name: "source", DatabaseConnectionID: in.DatabaseConnectionID} + })) + mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/schemaConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.SchemaConnection{ + {ID: publicSchemaID.String(), Name: "public", DatabaseConnectionID: dbConnID.String()}, + {ID: privateSchemaID.String(), Name: "private", DatabaseConnectionID: dbConnID.String()}, + }, + } + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + Include: "*", + }) + if err != nil { + t.Fatalf("expected no error (failures reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when multiple schemas are discovered without schemaNames") + } + if output.Error == "" { + t.Fatalf("expected an error message naming the discovered schemas") + } +} + +func TestConfigureDatabase_SchemaNamesSelectsSubset(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + dbConnID, _ := uuid.NewUUID() + databaseID, _ := uuid.NewUUID() + publicSchemaID, _ := uuid.NewUUID() + privateSchemaID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.DatabaseConnection{ + {ID: dbConnID.String(), Name: "source", EdgeConnectionID: edgeConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { + return http.StatusCreated, clients.Database{ID: databaseID.String(), Name: "source", DatabaseConnectionID: in.DatabaseConnectionID} + })) + mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/schemaConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.SchemaConnection{ + {ID: publicSchemaID.String(), Name: "public", DatabaseConnectionID: dbConnID.String()}, + {ID: privateSchemaID.String(), Name: "private", DatabaseConnectionID: dbConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/schemaMetadataConfigurations/batch", testutil.JsonHandlerInOut(func(r *http.Request, in []clients.SchemaMetadataConfiguration) (int, []clients.SchemaMetadataConfiguration) { + if len(in) != 1 || in[0].SchemaConnectionID != publicSchemaID.String() { + t.Fatalf("unexpected batch payload: %+v", in) + } + return http.StatusCreated, in + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + SchemaNames: []string{"public"}, + Include: "*", + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if len(output.SchemaConnections) != 1 || output.SchemaConnections[0].Name != "public" { + t.Fatalf("expected only the 'public' schema to be configured, got: %+v", output.SchemaConnections) + } +} + func TestConfigureDatabase_NoDatabaseConnectionsDiscovered(t *testing.T) { edgeConnID, _ := uuid.NewUUID() communityID, _ := uuid.NewUUID() @@ -122,6 +264,7 @@ func TestConfigureDatabase_NoDatabaseConnectionsDiscovered(t *testing.T) { CommunityID: communityID.String(), ParentSystemID: systemID.String(), OwnerIDs: []string{ownerID.String()}, + Include: "*", }) if err != nil { t.Fatalf("expected no error (failures reported via Output), got: %v", err) From cc694799f03ed74bae91b1f4541530c9c7a261a8 Mon Sep 17 00:00:00 2001 From: Dary Schaeken Date: Tue, 7 Jul 2026 14:10:44 +0200 Subject: [PATCH 06/10] fix: nil-slice output in list_capability_types query filtering A query matching capability types but zero connection types (or vice versa) left the unmatched side as a nil slice, which marshals to JSON null and fails the tool's own array-typed output schema. Initialize both filtered slices as empty instead of nil. --- pkg/tools/list_capability_types/tool.go | 4 ++-- pkg/tools/list_capability_types/tool_test.go | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/tools/list_capability_types/tool.go b/pkg/tools/list_capability_types/tool.go index 4717f4f..a27456f 100644 --- a/pkg/tools/list_capability_types/tool.go +++ b/pkg/tools/list_capability_types/tool.go @@ -59,13 +59,13 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { }, nil } - var matchedCapabilities []clients.CapabilityType + matchedCapabilities := []clients.CapabilityType{} for _, c := range capabilityTypes { if strings.Contains(strings.ToLower(c.ID), query) { matchedCapabilities = append(matchedCapabilities, c) } } - var matchedConnections []clients.ConnectionType + matchedConnections := []clients.ConnectionType{} for _, c := range connectionTypes { if strings.Contains(strings.ToLower(c.ID), query) { matchedConnections = append(matchedConnections, c) diff --git a/pkg/tools/list_capability_types/tool_test.go b/pkg/tools/list_capability_types/tool_test.go index baab63f..29f769c 100644 --- a/pkg/tools/list_capability_types/tool_test.go +++ b/pkg/tools/list_capability_types/tool_test.go @@ -101,6 +101,9 @@ func TestListCapabilityTypes_QueryFiltersAndKeepsManifest(t *testing.T) { if len(output.ConnectionTypes) != 0 { t.Fatalf("expected no connection types to match 'jdbc', got: %+v", output.ConnectionTypes) } + if output.ConnectionTypes == nil { + t.Fatalf("expected an empty slice, not nil — a nil slice marshals to JSON null, which fails the tool's array-typed output schema") + } } func TestListCapabilityTypes_InvalidEdgeSiteID(t *testing.T) { From 1f7b96f19e19518919466981a38cb567c2b4ba72 Mon Sep 17 00:00:00 2001 From: "volodymyr.bodnar" Date: Mon, 6 Jul 2026 21:41:03 -0400 Subject: [PATCH 07/10] feat: add non-duplicate integration MCP tools (edge, generic catalog, jobs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate the unique tools from feat/create-tool-for-integration onto the jdbc-ingestion branch, dropping everything that duplicates the existing lifecycle tools. Adds 23 tools: - edge_* capability/connection/job management (list/find/get/delete/run/ cancel/status-history) — the read/delete/run operations with no counterpart on this branch - catalog_generic_* GENERIC integration config, schedules, and jobs (/rest/catalog/1.0/genericIntegration) - jobs_find (/rest/jobs/v1/jobs) and list_integrations Dropped as duplicates: edge_create_connection, edge_find_connections, edge_test_connection, edge_create_capability, edge_get_job_status, jobs_get, edge_update_connection, edge_update_capability. Edge methods are consolidated into the existing per-entity clients; the duplicate capability/connection and job response types are merged into single EdgeCapability/EdgeConnection and Job types, and progressPercentage is int per job-api.yaml. --- job-api.yaml | 777 ++++++++++++++++++ pkg/clients/catalog_database_client.go | 9 +- pkg/clients/dgc_job_client.go | 121 ++- pkg/clients/dgc_job_client_test.go | 4 +- pkg/clients/edge_capability_client.go | 137 ++- pkg/clients/edge_connection_client.go | 85 +- pkg/clients/edge_job_client.go | 22 + pkg/clients/generic_integration_client.go | 279 +++++++ .../catalog_generic_add_schedule/tool.go | 53 ++ .../catalog_generic_add_schedule/tool_test.go | 72 ++ pkg/tools/catalog_generic_cancel_job/tool.go | 44 + .../catalog_generic_cancel_job/tool_test.go | 60 ++ .../catalog_generic_delete_config/tool.go | 43 + .../tool_test.go | 60 ++ .../catalog_generic_delete_schedule/tool.go | 44 + .../tool_test.go | 60 ++ .../catalog_generic_get_all_schedules/tool.go | 45 + .../tool_test.go | 64 ++ pkg/tools/catalog_generic_get_config/tool.go | 45 + .../catalog_generic_get_config/tool_test.go | 68 ++ .../catalog_generic_get_schedule/tool.go | 45 + .../catalog_generic_get_schedule/tool_test.go | 68 ++ pkg/tools/catalog_generic_get_schema/tool.go | 45 + .../catalog_generic_get_schema/tool_test.go | 63 ++ pkg/tools/catalog_generic_save_config/tool.go | 48 ++ .../catalog_generic_save_config/tool_test.go | 66 ++ pkg/tools/catalog_generic_start_job/tool.go | 48 ++ .../catalog_generic_start_job/tool_test.go | 64 ++ .../catalog_generic_update_schedule/tool.go | 53 ++ .../tool_test.go | 69 ++ pkg/tools/create_capability/tool.go | 4 +- pkg/tools/create_capability/tool_test.go | 20 +- pkg/tools/create_connection/tool.go | 4 +- pkg/tools/create_connection/tool_test.go | 44 +- pkg/tools/edge_cancel_job/tool.go | 43 + pkg/tools/edge_cancel_job/tool_test.go | 60 ++ pkg/tools/edge_delete_capability/tool.go | 43 + pkg/tools/edge_delete_capability/tool_test.go | 60 ++ pkg/tools/edge_delete_connection/tool.go | 43 + pkg/tools/edge_delete_connection/tool_test.go | 60 ++ pkg/tools/edge_find_capabilities/tool.go | 54 ++ pkg/tools/edge_find_capabilities/tool_test.go | 66 ++ pkg/tools/edge_get_capability/tool.go | 45 + pkg/tools/edge_get_capability/tool_test.go | 67 ++ pkg/tools/edge_get_connection/tool.go | 45 + pkg/tools/edge_get_connection/tool_test.go | 64 ++ pkg/tools/edge_get_job_status_history/tool.go | 48 ++ .../edge_get_job_status_history/tool_test.go | 70 ++ pkg/tools/edge_list_capabilities/tool.go | 74 ++ pkg/tools/edge_list_capabilities/tool_test.go | 144 ++++ pkg/tools/edge_list_connections/tool.go | 41 + pkg/tools/edge_list_connections/tool_test.go | 55 ++ pkg/tools/edge_run_capability/tool.go | 57 ++ pkg/tools/edge_run_capability/tool_test.go | 66 ++ pkg/tools/find_connections/tool.go | 2 +- pkg/tools/find_connections/tool_test.go | 8 +- pkg/tools/get_catalog_job_status/tool.go | 18 +- pkg/tools/jobs_find/tool.go | 51 ++ pkg/tools/jobs_find/tool_test.go | 74 ++ pkg/tools/list_integrations/tool.go | 205 +++++ pkg/tools/list_integrations/tool_test.go | 217 +++++ pkg/tools/register.go | 53 ++ pkg/tools/start_ingestion/tool.go | 2 +- pkg/tools/start_ingestion/tool_test.go | 8 +- 64 files changed, 4461 insertions(+), 115 deletions(-) create mode 100644 job-api.yaml create mode 100644 pkg/clients/generic_integration_client.go create mode 100644 pkg/tools/catalog_generic_add_schedule/tool.go create mode 100644 pkg/tools/catalog_generic_add_schedule/tool_test.go create mode 100644 pkg/tools/catalog_generic_cancel_job/tool.go create mode 100644 pkg/tools/catalog_generic_cancel_job/tool_test.go create mode 100644 pkg/tools/catalog_generic_delete_config/tool.go create mode 100644 pkg/tools/catalog_generic_delete_config/tool_test.go create mode 100644 pkg/tools/catalog_generic_delete_schedule/tool.go create mode 100644 pkg/tools/catalog_generic_delete_schedule/tool_test.go create mode 100644 pkg/tools/catalog_generic_get_all_schedules/tool.go create mode 100644 pkg/tools/catalog_generic_get_all_schedules/tool_test.go create mode 100644 pkg/tools/catalog_generic_get_config/tool.go create mode 100644 pkg/tools/catalog_generic_get_config/tool_test.go create mode 100644 pkg/tools/catalog_generic_get_schedule/tool.go create mode 100644 pkg/tools/catalog_generic_get_schedule/tool_test.go create mode 100644 pkg/tools/catalog_generic_get_schema/tool.go create mode 100644 pkg/tools/catalog_generic_get_schema/tool_test.go create mode 100644 pkg/tools/catalog_generic_save_config/tool.go create mode 100644 pkg/tools/catalog_generic_save_config/tool_test.go create mode 100644 pkg/tools/catalog_generic_start_job/tool.go create mode 100644 pkg/tools/catalog_generic_start_job/tool_test.go create mode 100644 pkg/tools/catalog_generic_update_schedule/tool.go create mode 100644 pkg/tools/catalog_generic_update_schedule/tool_test.go create mode 100644 pkg/tools/edge_cancel_job/tool.go create mode 100644 pkg/tools/edge_cancel_job/tool_test.go create mode 100644 pkg/tools/edge_delete_capability/tool.go create mode 100644 pkg/tools/edge_delete_capability/tool_test.go create mode 100644 pkg/tools/edge_delete_connection/tool.go create mode 100644 pkg/tools/edge_delete_connection/tool_test.go create mode 100644 pkg/tools/edge_find_capabilities/tool.go create mode 100644 pkg/tools/edge_find_capabilities/tool_test.go create mode 100644 pkg/tools/edge_get_capability/tool.go create mode 100644 pkg/tools/edge_get_capability/tool_test.go create mode 100644 pkg/tools/edge_get_connection/tool.go create mode 100644 pkg/tools/edge_get_connection/tool_test.go create mode 100644 pkg/tools/edge_get_job_status_history/tool.go create mode 100644 pkg/tools/edge_get_job_status_history/tool_test.go create mode 100644 pkg/tools/edge_list_capabilities/tool.go create mode 100644 pkg/tools/edge_list_capabilities/tool_test.go create mode 100644 pkg/tools/edge_list_connections/tool.go create mode 100644 pkg/tools/edge_list_connections/tool_test.go create mode 100644 pkg/tools/edge_run_capability/tool.go create mode 100644 pkg/tools/edge_run_capability/tool_test.go create mode 100644 pkg/tools/jobs_find/tool.go create mode 100644 pkg/tools/jobs_find/tool_test.go create mode 100644 pkg/tools/list_integrations/tool.go create mode 100644 pkg/tools/list_integrations/tool_test.go diff --git a/job-api.yaml b/job-api.yaml new file mode 100644 index 0000000..d781a61 --- /dev/null +++ b/job-api.yaml @@ -0,0 +1,777 @@ +--- +openapi: 3.0.3 +info: + description: This API provides the ability to retrieve job progress. + title: Collibra Job API + version: 1.0.0 + x-audience: public +servers: +- url: /rest/jobs/v1 +security: +- basicAuth: [] +- jwtAuth: [] +tags: +- description: Job endpoints provide the ability to retrieve job progress. + name: Jobs +paths: + /jobs: + get: + description: Returns jobs matching the given search criteria. Only parameters + that are specified in this request and have not null values are + used for filtering. All other parameters are ignored. + operationId: findJobs + parameters: + - description: "The match mode used to compare name. If the match\ + \ mode is EXACT the search is case-sensitive, otherwise the\ + \ search is case-insensitive." + explode: true + in: query + name: nameMatchMode + required: false + schema: + default: ANYWHERE + enum: + - START + - END + - ANYWHERE + - EXACT + example: EXACT + type: string + style: form + - description: The name of the job to search for. + explode: true + in: query + name: name + required: false + schema: + example: Sample job + type: string + style: form + - description: The list of results which the jobs should be searched for. + explode: true + in: query + name: result + required: false + schema: + items: + enum: + - NOT_SET + - SUCCESS + - COMPLETED_WITH_ERROR + - FAILURE + - ABORTED + example: SUCCESS + type: string + type: array + style: form + - description: The list of states which the jobs should be searched for. + explode: true + in: query + name: state + required: false + schema: + items: + enum: + - WAITING + - RUNNING + - COMPLETED + - FAILED + - DELETED + example: COMPLETED + type: string + type: array + style: form + - description: The list of types of the jobs. + explode: true + in: query + name: type + required: false + schema: + items: + example: IMPORT + type: string + type: array + style: form + - description: The ID of the user who created the searched job. + explode: true + in: query + name: user + required: false + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: form + - description: "The field that should be used as reference for sorting, \nin\ + \ addition to the CREATED_ON field in DESC order.\n" + explode: true + in: query + name: sortField + required: false + schema: + enum: + - NAME + - TYPE + - START_DATE + - END_DATE + - STATE + - RESULT + example: NAME + type: string + style: form + - description: The order of sorting. + explode: true + in: query + name: sortOrder + required: false + schema: + default: DESC + enum: + - ASC + - DESC + example: ASC + type: string + style: form + - description: "The cursor pointing to the first resource to be included in\ + \ the response. \nThis cursor cannot be created and must have been extracted\ + \ from a response returned by a previous API call.\n\nIf this parameter\ + \ is missing, the API returns the resources starting from the first available\ + \ resource, at index `0`.\n" + explode: true + in: query + name: cursor + required: false + schema: + type: string + style: form + - description: | + The maximum number of resources to retrieve. + + If not set, the default limit of `10` is be used. The maximum value for this parameter is `50`. + explode: true + in: query + name: pageSize + required: false + schema: + default: 10 + maximum: 50 + minimum: 1 + type: integer + style: form + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/JobPagedResponse" + description: The paged response with jobs matching filter criteria + "400": + content: + application/json: + example: + userMessage: The request is not valid. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The request is not valid. + "401": + content: + application/json: + example: + userMessage: Failed to authenticate the request through basic credentials. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The client is not authenticated. + "500": + content: + application/json: + example: + userMessage: Internal server error. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Internal server error. + summary: Find jobs + tags: + - Jobs + /jobs/{jobId}: + delete: + description: | + Soft deletes the Job identified by the given UUID. + Users can delete their own jobs and Admins can delete any job. + operationId: deleteJob + parameters: + - description: The ID of the job. + explode: false + in: path + name: jobId + required: true + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: simple + - description: An optional message for the job deletion. + explode: true + in: query + name: message + required: false + schema: + default: Deleted by user + maxLength: 128 + type: string + style: form + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/JobDelete" + description: | + Use the message query parameter instead. Deprecated, will be removed in 2026.06. + Note: The message defined in the query parameter takes precedence over the request body. + required: false + responses: + "204": + description: Job successfully deleted. + "400": + content: + application/json: + example: + userMessage: The request is not valid. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The request is not valid. + "401": + content: + application/json: + example: + userMessage: Failed to authenticate the request through basic credentials. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The client is not authenticated. + "404": + content: + application/json: + example: + userMessage: The resource could not be found. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Resource not found. + "500": + content: + application/json: + example: + userMessage: Internal server error. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Internal server error. + security: + - basicAuth: [] + - jwtAuth: [] + summary: Soft delete Job. + tags: + - Jobs + get: + description: Retrieves the Job identified by the given UUID. + operationId: getJob + parameters: + - description: The ID of the job. + explode: false + in: path + name: jobId + required: true + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: simple + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Job" + description: Job found. + "400": + content: + application/json: + example: + userMessage: The request is not valid. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The request is not valid. + "401": + content: + application/json: + example: + userMessage: Failed to authenticate the request through basic credentials. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The client is not authenticated. + "404": + content: + application/json: + example: + userMessage: The resource could not be found. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Resource not found. + "500": + content: + application/json: + example: + userMessage: Internal server error. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Internal server error. + summary: Get Job. + tags: + - Jobs +components: + parameters: + PageSizeQueryParam: + description: | + The maximum number of resources to retrieve. + + If not set, the default limit of `10` is be used. The maximum value for this parameter is `50`. + explode: true + in: query + name: pageSize + required: false + schema: + default: 10 + maximum: 50 + minimum: 1 + type: integer + style: form + CursorQueryParam: + description: "The cursor pointing to the first resource to be included in the\ + \ response. \nThis cursor cannot be created and must have been extracted from\ + \ a response returned by a previous API call.\n\nIf this parameter is missing,\ + \ the API returns the resources starting from the first available resource,\ + \ at index `0`.\n" + explode: true + in: query + name: cursor + required: false + schema: + type: string + style: form + UserQueryParam: + description: The ID of the user who created the searched job. + explode: true + in: query + name: user + required: false + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: form + JobIdQueryParam: + description: The ID of the job. + explode: false + in: path + name: jobId + required: true + schema: + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + style: simple + MessageQueryParam: + description: An optional message for the job deletion. + explode: true + in: query + name: message + required: false + schema: + default: Deleted by user + maxLength: 128 + type: string + style: form + NameQueryParam: + description: The name of the job to search for. + explode: true + in: query + name: name + required: false + schema: + example: Sample job + type: string + style: form + NameMatchModeQueryParam: + description: "The match mode used to compare name. If the match\ + \ mode is EXACT the search is case-sensitive, otherwise the search\ + \ is case-insensitive." + explode: true + in: query + name: nameMatchMode + required: false + schema: + default: ANYWHERE + enum: + - START + - END + - ANYWHERE + - EXACT + example: EXACT + type: string + style: form + ResultQueryParam: + description: The list of results which the jobs should be searched for. + explode: true + in: query + name: result + required: false + schema: + items: + enum: + - NOT_SET + - SUCCESS + - COMPLETED_WITH_ERROR + - FAILURE + - ABORTED + example: SUCCESS + type: string + type: array + style: form + SortFieldQueryParam: + description: "The field that should be used as reference for sorting, \nin addition\ + \ to the CREATED_ON field in DESC order.\n" + explode: true + in: query + name: sortField + required: false + schema: + enum: + - NAME + - TYPE + - START_DATE + - END_DATE + - STATE + - RESULT + example: NAME + type: string + style: form + SortOrderQueryParam: + description: The order of sorting. + explode: true + in: query + name: sortOrder + required: false + schema: + default: DESC + enum: + - ASC + - DESC + example: ASC + type: string + style: form + StateQueryParam: + description: The list of states which the jobs should be searched for. + explode: true + in: query + name: state + required: false + schema: + items: + enum: + - WAITING + - RUNNING + - COMPLETED + - FAILED + - DELETED + example: COMPLETED + type: string + type: array + style: form + TypeQueryParam: + description: The list of types of the jobs. + explode: true + in: query + name: type + required: false + schema: + items: + example: IMPORT + type: string + type: array + style: form + responses: + "400_Bad_Request": + content: + application/json: + example: + userMessage: The request is not valid. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The request is not valid. + "401_Unauthorized": + content: + application/json: + example: + userMessage: Failed to authenticate the request through basic credentials. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: The client is not authenticated. + "500_InternalServerError": + content: + application/json: + example: + userMessage: Internal server error. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Internal server error. + "404_Not_Found": + content: + application/json: + example: + userMessage: The resource could not be found. + schema: + $ref: "#/components/schemas/StandardErrorResponse" + description: Resource not found. + schemas: + JobDelete: + allOf: + - $ref: "#/components/schemas/JobMessage" + deprecated: true + description: "(Deprecated) Represents a body for delete job. Use the message\ + \ query parameter instead. Deprecated, will be removed in 2026.06." + example: + message: Import finished in 12s. + JobPagedResponse: + example: + nextCursor: Y3Vyc29yV2hLblRoZXJlSXNNb3JlRGF0YQo + results: + - endDate: 2017-07-21T17:32:28Z + lastModifiedBy: a073ff90-e7bc-4b35-ba90-c4d475f642fe + message: Import finished in 12s. + type: SAMPLE + createdOn: 2017-07-21T17:32:28Z + lastModifiedOn: 2017-07-21T17:32:28Z + result: SUCCESS + createdBy: 4d250cc5-e583-4640-9874-b93d82c7a6cb + name: Sample job. + progressPercentage: 50 + id: 495756bd-419c-4fea-beb4-8a70253d8079 + state: COMPLETED + user: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + selfManaged: false + startDate: 2017-07-21T17:32:28Z + - endDate: 2017-07-21T17:32:28Z + lastModifiedBy: a073ff90-e7bc-4b35-ba90-c4d475f642fe + message: Import finished in 12s. + type: SAMPLE + createdOn: 2017-07-21T17:32:28Z + lastModifiedOn: 2017-07-21T17:32:28Z + result: SUCCESS + createdBy: 4d250cc5-e583-4640-9874-b93d82c7a6cb + name: Sample job. + progressPercentage: 50 + id: 495756bd-419c-4fea-beb4-8a70253d8079 + state: COMPLETED + user: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + selfManaged: false + startDate: 2017-07-21T17:32:28Z + properties: + nextCursor: + description: "The cursor pointing to the next page. If the cursor is missing,\ + \ there are no additional pages of resources available after the current\ + \ one." + example: Y3Vyc29yV2hLblRoZXJlSXNNb3JlRGF0YQo + type: string + results: + description: The list of results. + items: + $ref: "#/components/schemas/Job" + type: array + type: object + Job: + allOf: + - properties: + id: + description: The UUID of the represented object (entity). + example: 495756bd-419c-4fea-beb4-8a70253d8079 + format: uuid + type: string + createdBy: + description: The UUID of the user that created this resource. + example: 4d250cc5-e583-4640-9874-b93d82c7a6cb + format: uuid + type: string + createdOn: + description: Date and time the job was created on. + example: 2017-07-21T17:32:28Z + format: date-time + type: string + lastModifiedBy: + description: The UUID of the user who modified this resource the last + time. + example: a073ff90-e7bc-4b35-ba90-c4d475f642fe + format: uuid + type: string + lastModifiedOn: + description: Date and time the job was modified on. + example: 2017-07-21T17:32:28Z + format: date-time + type: string + user: + description: The UUID of the user that initiated this job. + example: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + format: uuid + type: string + selfManaged: + default: false + description: "Whether this is a self managed job, processed outside of\ + \ DG." + example: false + type: boolean + required: + - id + type: object + - $ref: "#/components/schemas/JobEndDate" + - $ref: "#/components/schemas/JobMessage" + - $ref: "#/components/schemas/JobName" + - $ref: "#/components/schemas/JobProgressPercentage" + - $ref: "#/components/schemas/JobResult" + - $ref: "#/components/schemas/JobStartDate" + - $ref: "#/components/schemas/JobState" + - $ref: "#/components/schemas/JobType" + description: Represents a job. + example: + endDate: 2017-07-21T17:32:28Z + lastModifiedBy: a073ff90-e7bc-4b35-ba90-c4d475f642fe + message: Import finished in 12s. + type: SAMPLE + createdOn: 2017-07-21T17:32:28Z + lastModifiedOn: 2017-07-21T17:32:28Z + result: SUCCESS + createdBy: 4d250cc5-e583-4640-9874-b93d82c7a6cb + name: Sample job. + progressPercentage: 50 + id: 495756bd-419c-4fea-beb4-8a70253d8079 + state: COMPLETED + user: 387ca4ee-d48c-4902-b887-9cbfe11f4dac + selfManaged: false + startDate: 2017-07-21T17:32:28Z + JobEndDate: + properties: + endDate: + description: Date and time the job was ended on. + example: 2017-07-21T17:32:28Z + format: date-time + type: string + type: object + JobMessage: + properties: + message: + description: The message of the job. + example: Import finished in 12s. + type: string + type: object + JobName: + properties: + name: + description: The name of the job. + example: Sample job. + type: string + type: object + JobProgressPercentage: + properties: + progressPercentage: + description: The progress percentage of the job. + example: 50 + format: int32 + type: integer + type: object + JobResult: + properties: + result: + description: | + The result of the job. + + Possible values are: + * NOT_SET + * SUCCESS + * COMPLETED_WITH_ERROR + * FAILURE + * ABORTED + example: SUCCESS + type: string + type: object + JobStartDate: + properties: + startDate: + description: Date and time the job was started on. + example: 2017-07-21T17:32:28Z + format: date-time + type: string + type: object + JobState: + properties: + state: + description: | + The state of the job. + + Possible values are: + * WAITING + * RUNNING + * COMPLETED + * FAILED + * DELETED + example: COMPLETED + type: string + type: object + JobType: + properties: + type: + description: The type of the job. + example: SAMPLE + type: string + type: object + StandardErrorResponse: + example: + userMessage: The request is not valid. + titleMessage: Bad Request + errorContext: a073ff90-e7bc-4b35-ba90-c4d475f642fe + errorCode: invalidRequest + helpMessage: Check the request and try again. + statusCode: 400 + properties: + statusCode: + description: HTTP response code + example: 400 + type: integer + titleMessage: + description: The title of the error. + example: Bad Request + type: string + helpMessage: + description: A message that can help the user to understand the error. + example: Check the request and try again. + type: string + userMessage: + description: A message that can be shown to the user. + example: The request is not valid. + type: string + errorCode: + description: The error code. + example: invalidRequest + type: string + errorContext: + description: Span context ID related to the error. + example: a073ff90-e7bc-4b35-ba90-c4d475f642fe + type: string + required: + - errorCode + - statusCode + type: object + securitySchemes: + basicAuth: + description: Collibra REST API authentication using Basic Authentication. + scheme: basic + type: http + jwtAuth: + bearerFormat: JWT + description: Collibra REST API authentication using JSON Web Token. + scheme: bearer + type: http diff --git a/pkg/clients/catalog_database_client.go b/pkg/clients/catalog_database_client.go index 0a0c3ca..6a998a5 100644 --- a/pkg/clients/catalog_database_client.go +++ b/pkg/clients/catalog_database_client.go @@ -69,8 +69,9 @@ type SchemaMetadataConfiguration struct { SynchronizationRules []MetadataSynchronizationRule `json:"synchronizationRules,omitempty"` } -// Job represents an asynchronous job triggered by the database registration API. -type Job struct { +// CatalogJob represents an asynchronous job triggered by the database (catalog) +// registration API, e.g. POST /databases/{id}/synchronizeMetadata. +type CatalogJob struct { ID string `json:"id"` State string `json:"state,omitempty"` Result string `json:"result,omitempty"` @@ -157,13 +158,13 @@ func SetSchemaMetadataConfigurationsBatch(ctx context.Context, client *http.Clie // Database asset via POST /databases/{databaseId}/synchronizeMetadata. An empty // schemaConnectionIds list synchronizes all schemas that already have a // SchemaMetadataConfiguration. -func SynchronizeDatabaseMetadata(ctx context.Context, client *http.Client, databaseID string, schemaConnectionIDs []string) (*Job, error) { +func SynchronizeDatabaseMetadata(ctx context.Context, client *http.Client, databaseID string, schemaConnectionIDs []string) (*CatalogJob, error) { body, err := json.Marshal(DatabaseMetadataSynchronizationRequest{SchemaConnectionIDs: schemaConnectionIDs}) if err != nil { return nil, fmt.Errorf("starting ingestion: marshaling request: %w", err) } endpoint := catalogDatabaseBasePath + "/databases/" + databaseID + "/synchronizeMetadata" - var job Job + var job CatalogJob if err := doCatalogDatabasePostJSON(ctx, client, endpoint, body, http.StatusAccepted, &job); err != nil { return nil, err } diff --git a/pkg/clients/dgc_job_client.go b/pkg/clients/dgc_job_client.go index 7478732..f6dabdf 100644 --- a/pkg/clients/dgc_job_client.go +++ b/pkg/clients/dgc_job_client.go @@ -7,40 +7,117 @@ import ( "net/http" ) -// CatalogJob is a DGC job (e.g. the job returned by -// POST /rest/catalogDatabase/v1/databases/{id}/synchronizeMetadata), as returned by -// GET /rest/jobs/v1/jobs/{jobId}. This is the DGC-wide job resource — distinct from -// edge-management's per-Edge-site jobs (see GetJobStatusLog / get_job_status), which -// track Edge capability runs and connection tests instead. -type CatalogJob struct { - ID string `json:"id"` - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - State string `json:"state,omitempty"` - Result string `json:"result,omitempty"` - Message string `json:"message,omitempty"` - ProgressPercentage float64 `json:"progressPercentage,omitempty"` - StartDate string `json:"startDate,omitempty"` - EndDate string `json:"endDate,omitempty"` +// Job is a DGC job as returned by the Jobs API — GET /rest/jobs/v1/jobs and +// GET /rest/jobs/v1/jobs/{jobId}. Its fields mirror the Job schema in job-api.yaml. +// This is the DGC-wide job resource you poll for status (e.g. after triggering a +// catalog database sync, whose CatalogJob response carries the id to poll here). It is +// distinct from edge-management's per-Edge-site jobs (see GetJobStatusLog / +// get_job_status), which track Edge capability runs and connection tests instead. +type Job struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + State string `json:"state,omitempty"` + Result string `json:"result,omitempty"` + Message string `json:"message,omitempty"` + ProgressPercentage int `json:"progressPercentage,omitempty"` + StartDate string `json:"startDate,omitempty"` + EndDate string `json:"endDate,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + CreatedOn string `json:"createdOn,omitempty"` + LastModifiedBy string `json:"lastModifiedBy,omitempty"` + LastModifiedOn string `json:"lastModifiedOn,omitempty"` + User string `json:"user,omitempty"` + SelfManaged bool `json:"selfManaged,omitempty"` } -// GetCatalogJob fetches a DGC job's status via GET /rest/jobs/v1/jobs/{jobId}. This is -// the current, non-deprecated DGC job resource (the older /rest/2.0/jobs/{jobId} is +// JobPagedResponse is the paged response returned by GET /rest/jobs/v1/jobs. +type JobPagedResponse struct { + Results []Job `json:"results"` + NextCursor string `json:"nextCursor,omitempty"` +} + +type jobFindParams struct { + Name string `url:"name,omitempty"` + NameMatchMode string `url:"nameMatchMode,omitempty"` + Result []string `url:"result,omitempty"` + State []string `url:"state,omitempty"` + Type []string `url:"type,omitempty"` + User string `url:"user,omitempty"` + SortField string `url:"sortField,omitempty"` + SortOrder string `url:"sortOrder,omitempty"` + Cursor string `url:"cursor,omitempty"` + PageSize int `url:"pageSize,omitempty"` +} + +// GetJob fetches a DGC job's status via GET /rest/jobs/v1/jobs/{jobId}. This is the +// current, non-deprecated DGC job resource (the older /rest/2.0/jobs/{jobId} is // deprecated for removal, per its own javadoc, in favor of this one). -func GetCatalogJob(ctx context.Context, client *http.Client, jobID string) (*CatalogJob, error) { +func GetJob(ctx context.Context, client *http.Client, jobID string) (*Job, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, "/rest/jobs/v1/jobs/"+jobID, nil) if err != nil { - return nil, fmt.Errorf("getting catalog job %q: building request: %w", jobID, err) + return nil, fmt.Errorf("getting job %q: building request: %w", jobID, err) } body, err := executeRequest(client, req) if err != nil { - return nil, fmt.Errorf("getting catalog job %q: %w", jobID, err) + return nil, fmt.Errorf("getting job %q: %w", jobID, err) } - var result CatalogJob + var result Job if err := json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("getting catalog job %q: decoding response: %w", jobID, err) + return nil, fmt.Errorf("getting job %q: decoding response: %w", jobID, err) } return &result, nil } + +// FindJobs searches DGC jobs via GET /rest/jobs/v1/jobs, applying the given filters +// and pagination. All filters are optional. +func FindJobs( + ctx context.Context, + client *http.Client, + name string, + nameMatchMode string, + result []string, + state []string, + jobType []string, + user string, + sortField string, + sortOrder string, + cursor string, + pageSize int, +) (*JobPagedResponse, error) { + params := jobFindParams{ + Name: name, + NameMatchMode: nameMatchMode, + Result: result, + State: state, + Type: jobType, + User: user, + SortField: sortField, + SortOrder: sortOrder, + Cursor: cursor, + PageSize: pageSize, + } + + endpoint, err := buildUrl("/rest/jobs/v1/jobs", params) + if err != nil { + return nil, fmt.Errorf("finding jobs: building endpoint: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, fmt.Errorf("finding jobs: building request: %w", err) + } + + body, err := executeRequest(client, req) + if err != nil { + return nil, fmt.Errorf("finding jobs: %w", err) + } + + var resp JobPagedResponse + if err := json.Unmarshal(body, &resp); err != nil { + return nil, fmt.Errorf("finding jobs: decoding response: %w", err) + } + return &resp, nil +} diff --git a/pkg/clients/dgc_job_client_test.go b/pkg/clients/dgc_job_client_test.go index 42dc527..ed70db6 100644 --- a/pkg/clients/dgc_job_client_test.go +++ b/pkg/clients/dgc_job_client_test.go @@ -10,7 +10,7 @@ import ( "github.com/google/uuid" ) -func TestGetCatalogJob(t *testing.T) { +func TestGetJob(t *testing.T) { jobID, _ := uuid.NewUUID() handler := http.NewServeMux() @@ -23,7 +23,7 @@ func TestGetCatalogJob(t *testing.T) { defer server.Close() client := testutil.NewClient(server) - job, err := clients.GetCatalogJob(t.Context(), client, jobID.String()) + job, err := clients.GetJob(t.Context(), client, jobID.String()) if err != nil { t.Fatalf("expected no error, got: %v", err) } diff --git a/pkg/clients/edge_capability_client.go b/pkg/clients/edge_capability_client.go index 54be125..a3c5a80 100644 --- a/pkg/clients/edge_capability_client.go +++ b/pkg/clients/edge_capability_client.go @@ -9,17 +9,6 @@ import ( "net/http" ) -// Capability is the response shape returned by the Edge capability management API. -type Capability struct { - ID string `json:"id"` - Type any `json:"type,omitempty"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - EdgeSiteID string `json:"edgeSiteId"` - EdgeSiteName string `json:"edgeSiteName,omitempty"` - Parameters map[string]any `json:"parameters"` -} - // CapabilityRequest is the request body for creating or updating a capability via // POST /edge/api/rest/v2/capabilities or PUT /edge/api/rest/v2/capabilities/{id}. type CapabilityRequest struct { @@ -32,18 +21,18 @@ type CapabilityRequest struct { // CreateCapability creates a new Edge capability via POST /edge/api/rest/v2/capabilities. // The server assigns the capability id. -func CreateCapability(ctx context.Context, client *http.Client, request CapabilityRequest) (*Capability, error) { +func CreateCapability(ctx context.Context, client *http.Client, request CapabilityRequest) (*EdgeCapability, error) { return doCapabilityRequest(ctx, client, http.MethodPost, "/edge/api/rest/v2/capabilities", request) } // CreateOrUpdateCapability creates or updates an Edge capability with a known id via // PUT /edge/api/rest/v2/capabilities/{id}. -func CreateOrUpdateCapability(ctx context.Context, client *http.Client, capabilityID string, request CapabilityRequest) (*Capability, error) { +func CreateOrUpdateCapability(ctx context.Context, client *http.Client, capabilityID string, request CapabilityRequest) (*EdgeCapability, error) { endpoint := "/edge/api/rest/v2/capabilities/" + capabilityID return doCapabilityRequest(ctx, client, http.MethodPut, endpoint, request) } -func doCapabilityRequest(ctx context.Context, client *http.Client, method, endpoint string, request CapabilityRequest) (*Capability, error) { +func doCapabilityRequest(ctx context.Context, client *http.Client, method, endpoint string, request CapabilityRequest) (*EdgeCapability, error) { body, err := json.Marshal(request) if err != nil { return nil, fmt.Errorf("saving capability: marshaling request: %w", err) @@ -78,10 +67,128 @@ func doCapabilityRequest(ctx context.Context, client *http.Client, method, endpo } } - var result Capability + var result EdgeCapability if err := json.Unmarshal(respBody, &result); err != nil { return nil, fmt.Errorf("saving capability: decoding response: %w", err) } return &result, nil } + +// EdgeCapability is the full capability resource returned by the list/get/find +// endpoints. It is distinct from Capability (the create/update request-response +// shape) because these read endpoints expose the capability type as a structured +// object whose id is needed to filter integrations by platform. +type EdgeCapability struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + EdgeSiteId string `json:"edgeSiteId,omitempty"` + EdgeSiteName string `json:"edgeSiteName,omitempty"` + Type *EdgeCapabilityType `json:"type,omitempty"` + Parameters map[string]any `json:"parameters,omitempty"` +} + +type EdgeCapabilityType struct { + Id string `json:"id,omitempty"` +} + +type CapabilityFindRequest struct { + EdgeSiteId string `json:"edgeSiteId,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Parameters map[string]string `json:"parameters,omitempty"` +} + +type CapabilityRunRequest struct { + JobId string `json:"jobId,omitempty"` + InFastNamespace bool `json:"inFastNamespace,omitempty"` + Parameters map[string]any `json:"parameters,omitempty"` + WorkflowName string `json:"workflowName,omitempty"` +} + +// ListCapabilities lists all Edge capabilities via GET /edge/api/rest/v2/capabilities. +func ListCapabilities(ctx context.Context, client *http.Client) ([]EdgeCapability, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "/edge/api/rest/v2/capabilities", nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var capabilities []EdgeCapability + if err := json.Unmarshal(body, &capabilities); err != nil { + return nil, fmt.Errorf("failed to parse capabilities response: %w", err) + } + return capabilities, nil +} + +// FindCapabilities searches Edge capabilities via POST /edge/api/rest/v2/capabilities/find. +func FindCapabilities(ctx context.Context, client *http.Client, reqBody CapabilityFindRequest) ([]EdgeCapability, error) { + data, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "POST", "/edge/api/rest/v2/capabilities/find", bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var caps []EdgeCapability + if err := json.Unmarshal(body, &caps); err != nil { + return nil, fmt.Errorf("failed to parse capabilities response: %w", err) + } + return caps, nil +} + +// GetCapability fetches a single Edge capability via GET /edge/api/rest/v2/capabilities/{id}. +func GetCapability(ctx context.Context, client *http.Client, id string) (*EdgeCapability, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "/edge/api/rest/v2/capabilities/"+id, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var cap EdgeCapability + if err := json.Unmarshal(body, &cap); err != nil { + return nil, fmt.Errorf("failed to parse capability response: %w", err) + } + return &cap, nil +} + +// DeleteCapability deletes an Edge capability via DELETE /edge/api/rest/v2/capabilities/{id}. +func DeleteCapability(ctx context.Context, client *http.Client, id string) error { + req, err := http.NewRequestWithContext(ctx, "DELETE", "/edge/api/rest/v2/capabilities/"+id, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +// RunCapability triggers an Edge capability run via POST /edge/api/rest/v2/capabilities/{id}/run +// and returns the resulting job id. +func RunCapability(ctx context.Context, client *http.Client, id string, reqBody CapabilityRunRequest) (string, error) { + data, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "POST", "/edge/api/rest/v2/capabilities/"+id+"/run", bytes.NewBuffer(data)) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return "", err + } + var jobId string + if err := json.Unmarshal(body, &jobId); err != nil { + return "", fmt.Errorf("failed to parse run response: %w", err) + } + return jobId, nil +} diff --git a/pkg/clients/edge_connection_client.go b/pkg/clients/edge_connection_client.go index 101a35c..421ce55 100644 --- a/pkg/clients/edge_connection_client.go +++ b/pkg/clients/edge_connection_client.go @@ -10,17 +10,6 @@ import ( "net/http" ) -// Connection is the response shape returned by the Edge connection management API. -type Connection struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description,omitempty"` - TypeID string `json:"typeId,omitempty"` - EdgeSiteID string `json:"edgeSiteId"` - VaultID string `json:"vaultId,omitempty"` - Parameters map[string]any `json:"parameters"` -} - // ConnectionRequest is the request body for creating or updating a connection via // POST /edge/api/rest/v2/connections or PUT /edge/api/rest/v2/connections/{id}. type ConnectionRequest struct { @@ -34,13 +23,13 @@ type ConnectionRequest struct { // CreateConnection creates a new Edge connection via POST /edge/api/rest/v2/connections. // The server assigns the connection id. -func CreateConnection(ctx context.Context, client *http.Client, request ConnectionRequest) (*Connection, error) { +func CreateConnection(ctx context.Context, client *http.Client, request ConnectionRequest) (*EdgeConnection, error) { return doConnectionRequest(ctx, client, http.MethodPost, "/edge/api/rest/v2/connections", request) } // CreateOrUpdateConnection creates or updates an Edge connection with a known id via // PUT /edge/api/rest/v2/connections/{connectionId}. -func CreateOrUpdateConnection(ctx context.Context, client *http.Client, connectionID string, request ConnectionRequest) (*Connection, error) { +func CreateOrUpdateConnection(ctx context.Context, client *http.Client, connectionID string, request ConnectionRequest) (*EdgeConnection, error) { endpoint := "/edge/api/rest/v2/connections/" + connectionID return doConnectionRequest(ctx, client, http.MethodPut, endpoint, request) } @@ -56,7 +45,7 @@ type ConnectionFindRequest struct { // Useful for picking up a connection a user created manually (e.g. via the DGC/Edge UI, // for a driver file too large to pass through this tool) by name, instead of needing // its id. -func FindConnections(ctx context.Context, client *http.Client, request ConnectionFindRequest) ([]Connection, error) { +func FindConnections(ctx context.Context, client *http.Client, request ConnectionFindRequest) ([]EdgeConnection, error) { body, err := json.Marshal(request) if err != nil { return nil, fmt.Errorf("finding connections: marshaling request: %w", err) @@ -83,14 +72,14 @@ func FindConnections(ctx context.Context, client *http.Client, request Connectio return nil, fmt.Errorf("finding connections: unexpected status %d: %s", resp.StatusCode, string(respBody)) } - var result []Connection + var result []EdgeConnection if err := json.Unmarshal(respBody, &result); err != nil { return nil, fmt.Errorf("finding connections: decoding response: %w", err) } return result, nil } -func doConnectionRequest(ctx context.Context, client *http.Client, method, endpoint string, request ConnectionRequest) (*Connection, error) { +func doConnectionRequest(ctx context.Context, client *http.Client, method, endpoint string, request ConnectionRequest) (*EdgeConnection, error) { body, err := json.Marshal(request) if err != nil { return nil, fmt.Errorf("saving connection: marshaling request: %w", err) @@ -125,7 +114,7 @@ func doConnectionRequest(ctx context.Context, client *http.Client, method, endpo } } - var result Connection + var result EdgeConnection if err := json.Unmarshal(respBody, &result); err != nil { return nil, fmt.Errorf("saving connection: decoding response: %w", err) } @@ -188,3 +177,65 @@ func UploadFile(ctx context.Context, client *http.Client, filename string, conte return uri, nil } + +// EdgeConnection is the full connection resource returned by the list/get endpoints. +// It is distinct from Connection (the create/update request-response shape) because +// these read endpoints expose the connection type as a structured object. +type EdgeConnection struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + TypeId string `json:"typeId,omitempty"` + EdgeSiteId string `json:"edgeSiteId,omitempty"` + VaultId string `json:"vaultId,omitempty"` + ConnectionType *EdgeConnectionType `json:"connectionType,omitempty"` + Parameters map[string]any `json:"parameters,omitempty"` +} + +type EdgeConnectionType struct { + Id string `json:"id,omitempty"` +} + +// ListConnections lists all Edge connections via GET /edge/api/rest/v2/connections. +func ListConnections(ctx context.Context, client *http.Client) ([]EdgeConnection, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "/edge/api/rest/v2/connections", nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var connections []EdgeConnection + if err := json.Unmarshal(body, &connections); err != nil { + return nil, fmt.Errorf("failed to parse connections response: %w", err) + } + return connections, nil +} + +// GetConnection fetches a single Edge connection via GET /edge/api/rest/v2/connections/{id}. +func GetConnection(ctx context.Context, client *http.Client, id string) (*EdgeConnection, error) { + req, err := http.NewRequestWithContext(ctx, "GET", "/edge/api/rest/v2/connections/"+id, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var conn EdgeConnection + if err := json.Unmarshal(body, &conn); err != nil { + return nil, fmt.Errorf("failed to parse connection response: %w", err) + } + return &conn, nil +} + +// DeleteConnection deletes an Edge connection via DELETE /edge/api/rest/v2/connections/{id}. +func DeleteConnection(ctx context.Context, client *http.Client, id string) error { + req, err := http.NewRequestWithContext(ctx, "DELETE", "/edge/api/rest/v2/connections/"+id, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} diff --git a/pkg/clients/edge_job_client.go b/pkg/clients/edge_job_client.go index b899f12..00197fe 100644 --- a/pkg/clients/edge_job_client.go +++ b/pkg/clients/edge_job_client.go @@ -56,3 +56,25 @@ func GetJobStatusLog(ctx context.Context, client *http.Client, jobID string) (*J } return &result, nil } + +// CancelEdgeJob cancels a running Edge job via POST /edge/api/rest/v2/jobs/{id}/cancel. +func CancelEdgeJob(ctx context.Context, client *http.Client, id string) error { + req, err := http.NewRequestWithContext(ctx, "POST", "/edge/api/rest/v2/jobs/"+id+"/cancel", nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +// GetEdgeJobStatusHistory returns all status updates for an Edge job in reverse +// chronological order (most recent first) via +// GET /edge/api/rest/v2/jobs/{id}/statusLogHistory. +func GetEdgeJobStatusHistory(ctx context.Context, client *http.Client, id string) ([]JobStatusLog, error) { + endpoint := "/edge/api/rest/v2/jobs/" + id + "/statusLogHistory" + var result []JobStatusLog + if err := getJSON(ctx, client, endpoint, "getting job status history", &result); err != nil { + return nil, err + } + return result, nil +} diff --git a/pkg/clients/generic_integration_client.go b/pkg/clients/generic_integration_client.go new file mode 100644 index 0000000..456eb55 --- /dev/null +++ b/pkg/clients/generic_integration_client.go @@ -0,0 +1,279 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" +) + +const genericIntegrationBasePath = "/rest/catalog/1.0/genericIntegration" + +// --- Types --- + +type GenericConfiguration struct { + Id string `json:"id,omitempty"` + IngestibleId string `json:"ingestibleId,omitempty"` + Value string `json:"value,omitempty"` +} + +type SaveGenericConfigRequest struct { + Configuration string `json:"configuration"` +} + +type GenericSchedule struct { + Id int64 `json:"id,omitempty"` + CronExpression string `json:"cronExpression,omitempty"` + CronTimeZone string `json:"cronTimeZone,omitempty"` + LastRunTimeStamp int64 `json:"lastRunTimeStamp,omitempty"` + NextRunDateLongValue int64 `json:"nextRunDateLongValue,omitempty"` + CronJson string `json:"cronJson,omitempty"` + Workflow string `json:"workflow,omitempty"` +} + +type AddGenericScheduleRequest struct { + CronExpression string `json:"cronExpression"` + CronTimeZone string `json:"cronTimeZone"` + CronJson string `json:"cronJson,omitempty"` +} + +type ChangeGenericScheduleRequest struct { + CronExpression string `json:"cronExpression"` + CronTimeZone string `json:"cronTimeZone"` + CronJson string `json:"cronJson,omitempty"` +} + +type GenericJob struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + State string `json:"state,omitempty"` + Result string `json:"result,omitempty"` + Message string `json:"message,omitempty"` + ProgressPercentage float64 `json:"progressPercentage,omitempty"` + Cancelable bool `json:"cancelable,omitempty"` + StartDate int64 `json:"startDate,omitempty"` + EndDate int64 `json:"endDate,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + CreatedOn int64 `json:"createdOn,omitempty"` + UserId string `json:"userId,omitempty"` +} + +type genericScheduleParams struct { + Workflow string `url:"workflow,omitempty"` +} + +type startJobParams struct { + CloudIngestionJobId string `url:"cloudIngestionJobId,omitempty"` + Workflow string `url:"workflow,omitempty"` +} + +// --- Configuration functions --- + +func GetGenericConfig(ctx context.Context, client *http.Client, ingestibleId string) (*GenericConfiguration, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/configuration" + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var config GenericConfiguration + if err := json.Unmarshal(body, &config); err != nil { + return nil, fmt.Errorf("failed to parse config response: %w", err) + } + return &config, nil +} + +func SaveGenericConfig(ctx context.Context, client *http.Client, ingestibleId string, reqBody SaveGenericConfigRequest) (*GenericConfiguration, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/configuration" + data, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "PUT", endpoint, bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var config GenericConfiguration + if err := json.Unmarshal(body, &config); err != nil { + return nil, fmt.Errorf("failed to parse config response: %w", err) + } + return &config, nil +} + +func DeleteGenericConfig(ctx context.Context, client *http.Client, ingestibleId string) error { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/configuration" + req, err := http.NewRequestWithContext(ctx, "DELETE", endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +func GetGenericSchema(ctx context.Context, client *http.Client, ingestibleId string) (string, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/configuration/schema" + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return "", err + } + // Response is a JSON string value + var schema string + if err := json.Unmarshal(body, &schema); err != nil { + // If not a quoted string, return raw body + return string(body), nil + } + return schema, nil +} + +// --- Schedule functions --- + +func GetGenericSchedule(ctx context.Context, client *http.Client, ingestibleId string) (*GenericSchedule, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/schedule" + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var schedule GenericSchedule + if err := json.Unmarshal(body, &schedule); err != nil { + return nil, fmt.Errorf("failed to parse schedule response: %w", err) + } + return &schedule, nil +} + +func AddGenericSchedule(ctx context.Context, client *http.Client, ingestibleId string, workflow string, reqBody AddGenericScheduleRequest) (*GenericSchedule, error) { + params := genericScheduleParams{Workflow: workflow} + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/schedule", params) + if err != nil { + return nil, fmt.Errorf("failed to build endpoint: %w", err) + } + return doGenericScheduleRequest(ctx, client, "POST", endpoint, reqBody) +} + +func UpdateGenericSchedule(ctx context.Context, client *http.Client, ingestibleId string, workflow string, reqBody ChangeGenericScheduleRequest) (*GenericSchedule, error) { + params := genericScheduleParams{Workflow: workflow} + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/schedule", params) + if err != nil { + return nil, fmt.Errorf("failed to build endpoint: %w", err) + } + return doGenericScheduleRequest(ctx, client, "PUT", endpoint, reqBody) +} + +func DeleteGenericSchedule(ctx context.Context, client *http.Client, ingestibleId string, workflow string) error { + params := genericScheduleParams{Workflow: workflow} + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/schedule", params) + if err != nil { + return fmt.Errorf("failed to build endpoint: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "DELETE", endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +func GetAllGenericSchedules(ctx context.Context, client *http.Client, ingestibleId string) (*GenericSchedule, error) { + endpoint := genericIntegrationBasePath + "/" + ingestibleId + "/schedules" + req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var schedule GenericSchedule + if err := json.Unmarshal(body, &schedule); err != nil { + return nil, fmt.Errorf("failed to parse schedules response: %w", err) + } + return &schedule, nil +} + +func doGenericScheduleRequest(ctx context.Context, client *http.Client, method, endpoint string, reqBody any) (*GenericSchedule, error) { + data, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var schedule GenericSchedule + if err := json.Unmarshal(body, &schedule); err != nil { + return nil, fmt.Errorf("failed to parse schedule response: %w", err) + } + return &schedule, nil +} + +// --- Job functions --- + +func CancelGenericJob(ctx context.Context, client *http.Client, ingestibleId string, workflow string) error { + params := genericScheduleParams{Workflow: workflow} + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/cancel", params) + if err != nil { + return fmt.Errorf("failed to build endpoint: %w", err) + } + req, err := http.NewRequestWithContext(ctx, "DELETE", endpoint, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + _, err = executeRequest(client, req) + return err +} + +func StartGenericJob(ctx context.Context, client *http.Client, ingestibleId string, cloudIngestionJobId string, workflow string, runtimeArguments string) (*GenericJob, error) { + params := startJobParams{ + CloudIngestionJobId: cloudIngestionJobId, + Workflow: workflow, + } + endpoint, err := buildUrl(genericIntegrationBasePath+"/"+ingestibleId+"/run", params) + if err != nil { + return nil, fmt.Errorf("failed to build endpoint: %w", err) + } + + var bodyReader *bytes.Reader + if runtimeArguments != "" { + data, err := json.Marshal(runtimeArguments) + if err != nil { + return nil, fmt.Errorf("failed to marshal runtime arguments: %w", err) + } + bodyReader = bytes.NewReader(data) + } else { + bodyReader = bytes.NewReader([]byte{}) + } + + req, err := http.NewRequestWithContext(ctx, "POST", endpoint, bodyReader) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + body, err := executeRequest(client, req) + if err != nil { + return nil, err + } + var job GenericJob + if err := json.Unmarshal(body, &job); err != nil { + return nil, fmt.Errorf("failed to parse job response: %w", err) + } + return &job, nil +} \ No newline at end of file diff --git a/pkg/tools/catalog_generic_add_schedule/tool.go b/pkg/tools/catalog_generic_add_schedule/tool.go new file mode 100644 index 0000000..3942f07 --- /dev/null +++ b/pkg/tools/catalog_generic_add_schedule/tool.go @@ -0,0 +1,53 @@ +package catalog_generic_add_schedule + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + CronExpression string `json:"cronExpression" jsonschema:"the cron expression for the schedule (e.g. '0 2 * * *')"` + CronTimeZone string `json:"cronTimeZone" jsonschema:"the time zone for the cron schedule (e.g. 'UTC', 'America/New_York')"` + CronJson string `json:"cronJson,omitempty" jsonschema:"optional JSON representation of the cron schedule"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` +} + +type Output struct { + Schedule *clients.GenericSchedule `json:"schedule,omitempty" jsonschema:"the created schedule"` + Success bool `json:"success" jsonschema:"whether the schedule was added successfully"` + Error string `json:"error,omitempty" jsonschema:"error message if adding the schedule failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_add_schedule", + Description: "Adds a synchronization schedule to a GENERIC integration instance. Requires a cron expression and time zone.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schedule, err := clients.AddGenericSchedule(ctx, collibraClient, input.IngestibleId, input.Workflow, clients.AddGenericScheduleRequest{ + CronExpression: input.CronExpression, + CronTimeZone: input.CronTimeZone, + CronJson: input.CronJson, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to add schedule: %s", err.Error())}, nil + } + return Output{Schedule: schedule, Success: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_add_schedule/tool_test.go b/pkg/tools/catalog_generic_add_schedule/tool_test.go new file mode 100644 index 0000000..094d2f7 --- /dev/null +++ b/pkg/tools/catalog_generic_add_schedule/tool_test.go @@ -0,0 +1,72 @@ +package catalog_generic_add_schedule_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_generic_add_schedule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericAddSchedule(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddGenericScheduleRequest) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{Id: 1, CronExpression: in.CronExpression, CronTimeZone: in.CronTimeZone} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } + if output.Schedule == nil || output.Schedule.CronExpression != "0 2 * * *" { + t.Fatal("expected cron expression 0 2 * * *") + } +} + +func TestCatalogGenericAddScheduleAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusBadRequest, "bad request" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericAddScheduleInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "bad", + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + }) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_cancel_job/tool.go b/pkg/tools/catalog_generic_cancel_job/tool.go new file mode 100644 index 0000000..eb1abe9 --- /dev/null +++ b/pkg/tools/catalog_generic_cancel_job/tool.go @@ -0,0 +1,44 @@ +package catalog_generic_cancel_job + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the job was cancelled"` + Error string `json:"error,omitempty" jsonschema:"error message if cancellation failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_cancel_job", + Description: "Cancels the currently running synchronization job for a GENERIC integration instance.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + if err := clients.CancelGenericJob(ctx, collibraClient, input.IngestibleId, input.Workflow); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to cancel job: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_cancel_job/tool_test.go b/pkg/tools/catalog_generic_cancel_job/tool_test.go new file mode 100644 index 0000000..8065bc4 --- /dev/null +++ b/pkg/tools/catalog_generic_cancel_job/tool_test.go @@ -0,0 +1,60 @@ +package catalog_generic_cancel_job_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/catalog_generic_cancel_job" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericCancelJob(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/cancel", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusNoContent, nil + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestCatalogGenericCancelJobAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/cancel", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericCancelJobInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_delete_config/tool.go b/pkg/tools/catalog_generic_delete_config/tool.go new file mode 100644 index 0000000..6d3b62d --- /dev/null +++ b/pkg/tools/catalog_generic_delete_config/tool.go @@ -0,0 +1,43 @@ +package catalog_generic_delete_config + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance whose configuration to delete"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the configuration was deleted"` + Error string `json:"error,omitempty" jsonschema:"error message if deletion failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_delete_config", + Description: "Deletes the configuration for a GENERIC integration instance.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + if err := clients.DeleteGenericConfig(ctx, collibraClient, input.IngestibleId); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to delete configuration: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_delete_config/tool_test.go b/pkg/tools/catalog_generic_delete_config/tool_test.go new file mode 100644 index 0000000..d09e5b3 --- /dev/null +++ b/pkg/tools/catalog_generic_delete_config/tool_test.go @@ -0,0 +1,60 @@ +package catalog_generic_delete_config_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/catalog_generic_delete_config" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericDeleteConfig(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusNoContent, nil + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestCatalogGenericDeleteConfigAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericDeleteConfigInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_delete_schedule/tool.go b/pkg/tools/catalog_generic_delete_schedule/tool.go new file mode 100644 index 0000000..faaa67a --- /dev/null +++ b/pkg/tools/catalog_generic_delete_schedule/tool.go @@ -0,0 +1,44 @@ +package catalog_generic_delete_schedule + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the schedule was deleted"` + Error string `json:"error,omitempty" jsonschema:"error message if deletion failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_delete_schedule", + Description: "Deletes the synchronization schedule from a GENERIC integration instance.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + if err := clients.DeleteGenericSchedule(ctx, collibraClient, input.IngestibleId, input.Workflow); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to delete schedule: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_delete_schedule/tool_test.go b/pkg/tools/catalog_generic_delete_schedule/tool_test.go new file mode 100644 index 0000000..781e585 --- /dev/null +++ b/pkg/tools/catalog_generic_delete_schedule/tool_test.go @@ -0,0 +1,60 @@ +package catalog_generic_delete_schedule_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/catalog_generic_delete_schedule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericDeleteSchedule(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusNoContent, nil + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestCatalogGenericDeleteScheduleAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericDeleteScheduleInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_get_all_schedules/tool.go b/pkg/tools/catalog_generic_get_all_schedules/tool.go new file mode 100644 index 0000000..6aeb619 --- /dev/null +++ b/pkg/tools/catalog_generic_get_all_schedules/tool.go @@ -0,0 +1,45 @@ +package catalog_generic_get_all_schedules + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` +} + +type Output struct { + Schedule *clients.GenericSchedule `json:"schedule,omitempty" jsonschema:"the schedules for this integration"` + Found bool `json:"found" jsonschema:"whether schedules were found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_get_all_schedules", + Description: "Retrieves all synchronization schedules for a GENERIC integration instance.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schedule, err := clients.GetAllGenericSchedules(ctx, collibraClient, input.IngestibleId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get schedules: %s", err.Error())}, nil + } + return Output{Schedule: schedule, Found: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_get_all_schedules/tool_test.go b/pkg/tools/catalog_generic_get_all_schedules/tool_test.go new file mode 100644 index 0000000..6b5f185 --- /dev/null +++ b/pkg/tools/catalog_generic_get_all_schedules/tool_test.go @@ -0,0 +1,64 @@ +package catalog_generic_get_all_schedules_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_generic_get_all_schedules" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericGetAllSchedules(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedules", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{Id: 1, CronExpression: "0 2 * * *"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Schedule == nil || output.Schedule.Id != 1 { + t.Fatal("expected schedule id 1") + } +} + +func TestCatalogGenericGetAllSchedulesAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedules", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestCatalogGenericGetAllSchedulesInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_get_config/tool.go b/pkg/tools/catalog_generic_get_config/tool.go new file mode 100644 index 0000000..074a937 --- /dev/null +++ b/pkg/tools/catalog_generic_get_config/tool.go @@ -0,0 +1,45 @@ +package catalog_generic_get_config + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` +} + +type Output struct { + Config *clients.GenericConfiguration `json:"config,omitempty" jsonschema:"the integration configuration"` + Found bool `json:"found" jsonschema:"whether a configuration was found for this integration"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_get_config", + Description: "Retrieves the configuration for a GENERIC integration instance (e.g. Databricks Unity Catalog, Dataplex). The configuration value is a JSON string specific to the integration type.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + config, err := clients.GetGenericConfig(ctx, collibraClient, input.IngestibleId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get configuration: %s", err.Error())}, nil + } + return Output{Config: config, Found: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_get_config/tool_test.go b/pkg/tools/catalog_generic_get_config/tool_test.go new file mode 100644 index 0000000..c3f0e8e --- /dev/null +++ b/pkg/tools/catalog_generic_get_config/tool_test.go @@ -0,0 +1,68 @@ +package catalog_generic_get_config_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_generic_get_config" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericGetConfig(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericConfiguration) { + return http.StatusOK, clients.GenericConfiguration{ + Id: "cfg-1", + IngestibleId: "00000000-0000-0000-0000-000000000001", + Value: `{"key":"value"}`, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Config == nil || output.Config.Id != "cfg-1" { + t.Fatal("expected config id cfg-1") + } +} + +func TestCatalogGenericGetConfigNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000002/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestCatalogGenericGetConfigInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_get_schedule/tool.go b/pkg/tools/catalog_generic_get_schedule/tool.go new file mode 100644 index 0000000..f9386d4 --- /dev/null +++ b/pkg/tools/catalog_generic_get_schedule/tool.go @@ -0,0 +1,45 @@ +package catalog_generic_get_schedule + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` +} + +type Output struct { + Schedule *clients.GenericSchedule `json:"schedule,omitempty" jsonschema:"the integration schedule"` + Found bool `json:"found" jsonschema:"whether a schedule was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_get_schedule", + Description: "Retrieves the synchronization schedule for a GENERIC integration instance. Returns the cron expression, time zone, and timing of the last/next run.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schedule, err := clients.GetGenericSchedule(ctx, collibraClient, input.IngestibleId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get schedule: %s", err.Error())}, nil + } + return Output{Schedule: schedule, Found: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_get_schedule/tool_test.go b/pkg/tools/catalog_generic_get_schedule/tool_test.go new file mode 100644 index 0000000..af54493 --- /dev/null +++ b/pkg/tools/catalog_generic_get_schedule/tool_test.go @@ -0,0 +1,68 @@ +package catalog_generic_get_schedule_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_generic_get_schedule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericGetSchedule(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{ + Id: 1, + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Schedule == nil || output.Schedule.CronExpression != "0 2 * * *" { + t.Fatal("expected cron expression 0 2 * * *") + } +} + +func TestCatalogGenericGetScheduleNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000002/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestCatalogGenericGetScheduleInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_get_schema/tool.go b/pkg/tools/catalog_generic_get_schema/tool.go new file mode 100644 index 0000000..6e8afea --- /dev/null +++ b/pkg/tools/catalog_generic_get_schema/tool.go @@ -0,0 +1,45 @@ +package catalog_generic_get_schema + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` +} + +type Output struct { + Schema string `json:"schema,omitempty" jsonschema:"the data schema as a JSON string; describes the valid configuration fields for this integration"` + Found bool `json:"found" jsonschema:"whether a schema was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_get_schema", + Description: "Retrieves the data schema for a GENERIC integration instance. The schema describes the valid configuration fields that can be used with catalog_generic_save_config.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schema, err := clients.GetGenericSchema(ctx, collibraClient, input.IngestibleId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get schema: %s", err.Error())}, nil + } + return Output{Schema: schema, Found: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_get_schema/tool_test.go b/pkg/tools/catalog_generic_get_schema/tool_test.go new file mode 100644 index 0000000..5a8f0ce --- /dev/null +++ b/pkg/tools/catalog_generic_get_schema/tool_test.go @@ -0,0 +1,63 @@ +package catalog_generic_get_schema_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/catalog_generic_get_schema" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericGetSchema(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration/schema", testutil.StringHandlerOut(func(r *http.Request) (int, string) { + return http.StatusOK, `{"type":"object","properties":{"catalog":{"type":"string"}}}` + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Schema == "" { + t.Fatal("expected non-empty schema") + } +} + +func TestCatalogGenericGetSchemaNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000002/configuration/schema", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestCatalogGenericGetSchemaInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_save_config/tool.go b/pkg/tools/catalog_generic_save_config/tool.go new file mode 100644 index 0000000..1864c40 --- /dev/null +++ b/pkg/tools/catalog_generic_save_config/tool.go @@ -0,0 +1,48 @@ +package catalog_generic_save_config + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + Configuration string `json:"configuration" jsonschema:"the configuration JSON string to save"` +} + +type Output struct { + Config *clients.GenericConfiguration `json:"config,omitempty" jsonschema:"the saved configuration"` + Success bool `json:"success" jsonschema:"whether the configuration was saved successfully"` + Error string `json:"error,omitempty" jsonschema:"error message if saving failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_save_config", + Description: "Creates or updates the configuration for a GENERIC integration instance. The configuration is a JSON string whose schema depends on the integration type. Use catalog_generic_get_schema to discover valid fields.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + config, err := clients.SaveGenericConfig(ctx, collibraClient, input.IngestibleId, clients.SaveGenericConfigRequest{ + Configuration: input.Configuration, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to save configuration: %s", err.Error())}, nil + } + return Output{Config: config, Success: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_save_config/tool_test.go b/pkg/tools/catalog_generic_save_config/tool_test.go new file mode 100644 index 0000000..79565a9 --- /dev/null +++ b/pkg/tools/catalog_generic_save_config/tool_test.go @@ -0,0 +1,66 @@ +package catalog_generic_save_config_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_generic_save_config" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericSaveConfig(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerInOut(func(r *http.Request, in clients.SaveGenericConfigRequest) (int, clients.GenericConfiguration) { + return http.StatusOK, clients.GenericConfiguration{Id: "cfg-1", IngestibleId: "00000000-0000-0000-0000-000000000001", Value: in.Configuration} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + Configuration: `{"catalog":"my_catalog"}`, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } + if output.Config == nil || output.Config.Id != "cfg-1" { + t.Fatal("expected config id cfg-1") + } +} + +func TestCatalogGenericSaveConfigAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/configuration", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusBadRequest, "bad request" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + Configuration: "{}", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericSaveConfigInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad", Configuration: "{}"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_start_job/tool.go b/pkg/tools/catalog_generic_start_job/tool.go new file mode 100644 index 0000000..4a35cea --- /dev/null +++ b/pkg/tools/catalog_generic_start_job/tool.go @@ -0,0 +1,48 @@ +package catalog_generic_start_job + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + CloudIngestionJobId string `json:"cloudIngestionJobId,omitempty" jsonschema:"optional cloud ingestion job ID to resume"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` + RuntimeArguments string `json:"runtimeArguments,omitempty" jsonschema:"optional runtime arguments as a string"` +} + +type Output struct { + Job *clients.GenericJob `json:"job,omitempty" jsonschema:"the started job details"` + Success bool `json:"success" jsonschema:"whether the job was started"` + Error string `json:"error,omitempty" jsonschema:"error message if starting failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_start_job", + Description: "Starts a synchronization job for a GENERIC integration instance.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + job, err := clients.StartGenericJob(ctx, collibraClient, input.IngestibleId, input.CloudIngestionJobId, input.Workflow, input.RuntimeArguments) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to start job: %s", err.Error())}, nil + } + return Output{Job: job, Success: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_start_job/tool_test.go b/pkg/tools/catalog_generic_start_job/tool_test.go new file mode 100644 index 0000000..3551df6 --- /dev/null +++ b/pkg/tools/catalog_generic_start_job/tool_test.go @@ -0,0 +1,64 @@ +package catalog_generic_start_job_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_generic_start_job" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericStartJob(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/run", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericJob) { + return http.StatusAccepted, clients.GenericJob{Id: "job-1", State: "RUNNING"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } + if output.Job == nil || output.Job.Id != "job-1" { + t.Fatal("expected job id job-1") + } +} + +func TestCatalogGenericStartJobAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/run", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusConflict, "job already running" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericStartJobInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{IngestibleId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/catalog_generic_update_schedule/tool.go b/pkg/tools/catalog_generic_update_schedule/tool.go new file mode 100644 index 0000000..6f42055 --- /dev/null +++ b/pkg/tools/catalog_generic_update_schedule/tool.go @@ -0,0 +1,53 @@ +package catalog_generic_update_schedule + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + CronExpression string `json:"cronExpression" jsonschema:"the updated cron expression (e.g. '0 3 * * *')"` + CronTimeZone string `json:"cronTimeZone" jsonschema:"the updated time zone for the cron schedule"` + CronJson string `json:"cronJson,omitempty" jsonschema:"optional JSON representation of the cron schedule"` + Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` +} + +type Output struct { + Schedule *clients.GenericSchedule `json:"schedule,omitempty" jsonschema:"the updated schedule"` + Success bool `json:"success" jsonschema:"whether the schedule was updated successfully"` + Error string `json:"error,omitempty" jsonschema:"error message if the update failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "catalog_generic_update_schedule", + Description: "Updates the synchronization schedule for a GENERIC integration instance. Creates the schedule if it does not exist.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("ingestibleId", input.IngestibleId); err != nil { + return Output{}, err + } + schedule, err := clients.UpdateGenericSchedule(ctx, collibraClient, input.IngestibleId, input.Workflow, clients.ChangeGenericScheduleRequest{ + CronExpression: input.CronExpression, + CronTimeZone: input.CronTimeZone, + CronJson: input.CronJson, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to update schedule: %s", err.Error())}, nil + } + return Output{Schedule: schedule, Success: true}, nil + } +} diff --git a/pkg/tools/catalog_generic_update_schedule/tool_test.go b/pkg/tools/catalog_generic_update_schedule/tool_test.go new file mode 100644 index 0000000..79e8ef6 --- /dev/null +++ b/pkg/tools/catalog_generic_update_schedule/tool_test.go @@ -0,0 +1,69 @@ +package catalog_generic_update_schedule_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/catalog_generic_update_schedule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestCatalogGenericUpdateSchedule(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ChangeGenericScheduleRequest) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{Id: 1, CronExpression: in.CronExpression} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + CronExpression: "0 3 * * *", + CronTimeZone: "UTC", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestCatalogGenericUpdateScheduleAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusBadRequest, "bad" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "00000000-0000-0000-0000-000000000001", + CronExpression: "0 3 * * *", + CronTimeZone: "UTC", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestCatalogGenericUpdateScheduleInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + IngestibleId: "bad", + CronExpression: "0 3 * * *", + CronTimeZone: "UTC", + }) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/create_capability/tool.go b/pkg/tools/create_capability/tool.go index f64c781..eaadb3b 100644 --- a/pkg/tools/create_capability/tool.go +++ b/pkg/tools/create_capability/tool.go @@ -24,7 +24,7 @@ type Input struct { } type Output struct { - Capability *clients.Capability `json:"capability,omitempty" jsonschema:"The created or updated capability."` + Capability *clients.EdgeCapability `json:"capability,omitempty" jsonschema:"The created or updated capability."` Success bool `json:"success" jsonschema:"Whether the capability was saved successfully."` Error string `json:"error,omitempty" jsonschema:"Error message if saving the capability failed."` } @@ -62,7 +62,7 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { Parameters: parameters, } - var capability *clients.Capability + var capability *clients.EdgeCapability var err error if input.CapabilityID != "" { capability, err = clients.CreateOrUpdateCapability(ctx, collibraClient, input.CapabilityID, request) diff --git a/pkg/tools/create_capability/tool_test.go b/pkg/tools/create_capability/tool_test.go index 6b3fd15..f183c76 100644 --- a/pkg/tools/create_capability/tool_test.go +++ b/pkg/tools/create_capability/tool_test.go @@ -16,14 +16,14 @@ func TestCreateCapability_Create(t *testing.T) { capID, _ := uuid.NewUUID() handler := http.NewServeMux() - handler.Handle("POST /edge/api/rest/v2/capabilities", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityRequest) (int, clients.Capability) { + handler.Handle("POST /edge/api/rest/v2/capabilities", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityRequest) (int, clients.EdgeCapability) { if in.TypeID != "jdbc-ingestion" { t.Fatalf("unexpected typeId: %s", in.TypeID) } - return http.StatusCreated, clients.Capability{ - ID: capID.String(), + return http.StatusCreated, clients.EdgeCapability{ + Id: capID.String(), Name: in.Name, - EdgeSiteID: in.EdgeSiteID, + EdgeSiteId: in.EdgeSiteID, Parameters: in.Parameters, } })) @@ -44,8 +44,8 @@ func TestCreateCapability_Create(t *testing.T) { if !output.Success { t.Fatalf("expected success, got error: %s", output.Error) } - if output.Capability.ID != capID.String() { - t.Fatalf("expected id %s, got %s", capID.String(), output.Capability.ID) + if output.Capability.Id != capID.String() { + t.Fatalf("expected id %s, got %s", capID.String(), output.Capability.Id) } } @@ -54,11 +54,11 @@ func TestCreateCapability_UpdateWithKnownID(t *testing.T) { capID, _ := uuid.NewUUID() handler := http.NewServeMux() - handler.Handle("PUT /edge/api/rest/v2/capabilities/"+capID.String(), testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityRequest) (int, clients.Capability) { - return http.StatusOK, clients.Capability{ - ID: capID.String(), + handler.Handle("PUT /edge/api/rest/v2/capabilities/"+capID.String(), testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityRequest) (int, clients.EdgeCapability) { + return http.StatusOK, clients.EdgeCapability{ + Id: capID.String(), Name: in.Name, - EdgeSiteID: in.EdgeSiteID, + EdgeSiteId: in.EdgeSiteID, Parameters: in.Parameters, } })) diff --git a/pkg/tools/create_connection/tool.go b/pkg/tools/create_connection/tool.go index f595afe..855441f 100644 --- a/pkg/tools/create_connection/tool.go +++ b/pkg/tools/create_connection/tool.go @@ -40,7 +40,7 @@ type AdditionalProperty struct { } type Output struct { - Connection *clients.Connection `json:"connection,omitempty" jsonschema:"The created or updated connection."` + Connection *clients.EdgeConnection `json:"connection,omitempty" jsonschema:"The created or updated connection."` Success bool `json:"success" jsonschema:"Whether the connection was saved successfully."` Error string `json:"error,omitempty" jsonschema:"Error message if saving the connection failed."` } @@ -106,7 +106,7 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { Parameters: parameters, } - var connection *clients.Connection + var connection *clients.EdgeConnection var err error if input.ConnectionID != "" { connection, err = clients.CreateOrUpdateConnection(ctx, collibraClient, input.ConnectionID, request) diff --git a/pkg/tools/create_connection/tool_test.go b/pkg/tools/create_connection/tool_test.go index 7d21b96..8213310 100644 --- a/pkg/tools/create_connection/tool_test.go +++ b/pkg/tools/create_connection/tool_test.go @@ -16,18 +16,18 @@ func TestCreateConnection_Create(t *testing.T) { connID, _ := uuid.NewUUID() handler := http.NewServeMux() - handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { if in.Name != "local-postgres-source" { t.Fatalf("unexpected name: %s", in.Name) } if in.EdgeSiteID != siteID.String() { t.Fatalf("unexpected edgeSiteId: %s", in.EdgeSiteID) } - return http.StatusCreated, clients.Connection{ - ID: connID.String(), + return http.StatusCreated, clients.EdgeConnection{ + Id: connID.String(), Name: in.Name, - TypeID: in.TypeID, - EdgeSiteID: in.EdgeSiteID, + TypeId: in.TypeID, + EdgeSiteId: in.EdgeSiteID, Parameters: in.Parameters, } })) @@ -48,8 +48,8 @@ func TestCreateConnection_Create(t *testing.T) { if !output.Success { t.Fatalf("expected success, got error: %s", output.Error) } - if output.Connection.ID != connID.String() { - t.Fatalf("expected id %s, got %s", connID.String(), output.Connection.ID) + if output.Connection.Id != connID.String() { + t.Fatalf("expected id %s, got %s", connID.String(), output.Connection.Id) } } @@ -58,12 +58,12 @@ func TestCreateConnection_UpdateWithKnownID(t *testing.T) { connID, _ := uuid.NewUUID() handler := http.NewServeMux() - handler.Handle("PUT /edge/api/rest/v2/connections/"+connID.String(), testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { - return http.StatusOK, clients.Connection{ - ID: connID.String(), + handler.Handle("PUT /edge/api/rest/v2/connections/"+connID.String(), testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { + return http.StatusOK, clients.EdgeConnection{ + Id: connID.String(), Name: in.Name, - TypeID: in.TypeID, - EdgeSiteID: in.EdgeSiteID, + TypeId: in.TypeID, + EdgeSiteId: in.EdgeSiteID, Parameters: in.Parameters, } })) @@ -85,8 +85,8 @@ func TestCreateConnection_UpdateWithKnownID(t *testing.T) { if !output.Success { t.Fatalf("expected success, got error: %s", output.Error) } - if output.Connection.ID != connID.String() { - t.Fatalf("expected id %s, got %s", connID.String(), output.Connection.ID) + if output.Connection.Id != connID.String() { + t.Fatalf("expected id %s, got %s", connID.String(), output.Connection.Id) } } @@ -111,7 +111,7 @@ func TestCreateConnection_AdditionalProperties(t *testing.T) { connID, _ := uuid.NewUUID() handler := http.NewServeMux() - handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { props, ok := in.Parameters["connection-properties"].([]any) if !ok || len(props) != 2 { t.Fatalf("expected 2 connection-properties entries, got: %#v", in.Parameters["connection-properties"]) @@ -124,7 +124,7 @@ func TestCreateConnection_AdditionalProperties(t *testing.T) { if !ok || keyFile["name"] != "private_key_file" || keyFile["type"] != "file" || keyFile["value"] != "artifact://uuid/key.p8" || keyFile["secret"] != true { t.Fatalf("unexpected private_key_file entry: %#v", props[1]) } - return http.StatusCreated, clients.Connection{ID: connID.String(), Name: in.Name, EdgeSiteID: in.EdgeSiteID, Parameters: in.Parameters} + return http.StatusCreated, clients.EdgeConnection{Id: connID.String(), Name: in.Name, EdgeSiteId: in.EdgeSiteID, Parameters: in.Parameters} })) server := httptest.NewServer(handler) @@ -156,11 +156,11 @@ func TestCreateConnection_AdditionalProperties_CustomKey(t *testing.T) { siteID, _ := uuid.NewUUID() handler := http.NewServeMux() - handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { if _, ok := in.Parameters["additional-parameters"]; !ok { t.Fatalf("expected additional-parameters key, got parameters: %#v", in.Parameters) } - return http.StatusCreated, clients.Connection{Parameters: in.Parameters} + return http.StatusCreated, clients.EdgeConnection{Parameters: in.Parameters} })) server := httptest.NewServer(handler) @@ -192,14 +192,14 @@ func TestCreateConnection_PreUploadedDriverJar(t *testing.T) { const uploadedJarURI = "jar://test-uuid/driver.jar" handler := http.NewServeMux() - handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.Connection) { + handler.Handle("POST /edge/api/rest/v2/connections", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionRequest) (int, clients.EdgeConnection) { if in.Parameters["driver-jar"] != uploadedJarURI { t.Fatalf("expected driver-jar %q, got %v", uploadedJarURI, in.Parameters["driver-jar"]) } - return http.StatusCreated, clients.Connection{ - ID: connID.String(), + return http.StatusCreated, clients.EdgeConnection{ + Id: connID.String(), Name: in.Name, - EdgeSiteID: in.EdgeSiteID, + EdgeSiteId: in.EdgeSiteID, Parameters: in.Parameters, } })) diff --git a/pkg/tools/edge_cancel_job/tool.go b/pkg/tools/edge_cancel_job/tool.go new file mode 100644 index 0000000..1e78e77 --- /dev/null +++ b/pkg/tools/edge_cancel_job/tool.go @@ -0,0 +1,43 @@ +package edge_cancel_job + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the Edge job to cancel"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the cancellation request was accepted"` + Error string `json:"error,omitempty" jsonschema:"error message if cancellation failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_cancel_job", + Description: "Stops a running Edge job by its UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + if err := clients.CancelEdgeJob(ctx, collibraClient, input.Id); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to cancel job: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/edge_cancel_job/tool_test.go b/pkg/tools/edge_cancel_job/tool_test.go new file mode 100644 index 0000000..b4e9cef --- /dev/null +++ b/pkg/tools/edge_cancel_job/tool_test.go @@ -0,0 +1,60 @@ +package edge_cancel_job_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/edge_cancel_job" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeCancelJob(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/jobs/00000000-0000-0000-0000-000000000001/cancel", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestEdgeCancelJobAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/jobs/00000000-0000-0000-0000-000000000001/cancel", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "job not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestEdgeCancelJobInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_delete_capability/tool.go b/pkg/tools/edge_delete_capability/tool.go new file mode 100644 index 0000000..42595fb --- /dev/null +++ b/pkg/tools/edge_delete_capability/tool.go @@ -0,0 +1,43 @@ +package edge_delete_capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the capability to delete"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the capability was deleted"` + Error string `json:"error,omitempty" jsonschema:"error message if deletion failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_delete_capability", + Description: "Deletes an Edge capability by its UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + if err := clients.DeleteCapability(ctx, collibraClient, input.Id); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to delete capability: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/edge_delete_capability/tool_test.go b/pkg/tools/edge_delete_capability/tool_test.go new file mode 100644 index 0000000..b936c08 --- /dev/null +++ b/pkg/tools/edge_delete_capability/tool_test.go @@ -0,0 +1,60 @@ +package edge_delete_capability_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/edge_delete_capability" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeDeleteCapability(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestEdgeDeleteCapabilityAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000002", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestEdgeDeleteCapabilityInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_delete_connection/tool.go b/pkg/tools/edge_delete_connection/tool.go new file mode 100644 index 0000000..5e7e03e --- /dev/null +++ b/pkg/tools/edge_delete_connection/tool.go @@ -0,0 +1,43 @@ +package edge_delete_connection + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + ConnectionId string `json:"connectionId" jsonschema:"the UUID of the connection to delete"` +} + +type Output struct { + Success bool `json:"success" jsonschema:"whether the connection was deleted"` + Error string `json:"error,omitempty" jsonschema:"error message if deletion failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_delete_connection", + Description: "Deletes an Edge connection by its UUID.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("connectionId", input.ConnectionId); err != nil { + return Output{}, err + } + if err := clients.DeleteConnection(ctx, collibraClient, input.ConnectionId); err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to delete connection: %s", err.Error())}, nil + } + return Output{Success: true}, nil + } +} diff --git a/pkg/tools/edge_delete_connection/tool_test.go b/pkg/tools/edge_delete_connection/tool_test.go new file mode 100644 index 0000000..49e79a8 --- /dev/null +++ b/pkg/tools/edge_delete_connection/tool_test.go @@ -0,0 +1,60 @@ +package edge_delete_connection_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/edge_delete_connection" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeDeleteConnection(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + ConnectionId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } +} + +func TestEdgeDeleteConnectionAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + ConnectionId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } +} + +func TestEdgeDeleteConnectionInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ConnectionId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_find_capabilities/tool.go b/pkg/tools/edge_find_capabilities/tool.go new file mode 100644 index 0000000..21ede57 --- /dev/null +++ b/pkg/tools/edge_find_capabilities/tool.go @@ -0,0 +1,54 @@ +package edge_find_capabilities + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + EdgeSiteId string `json:"edgeSiteId,omitempty" jsonschema:"optional UUID of the edge site to filter capabilities by"` + Labels map[string]string `json:"labels,omitempty" jsonschema:"optional label key-value pairs to filter by; multiple labels are ANDed"` + Parameters map[string]string `json:"parameters,omitempty" jsonschema:"optional parameter key-value pairs to filter by (e.g. connection UUID)"` +} + +type Output struct { + Capabilities []clients.EdgeCapability `json:"capabilities" jsonschema:"list of matching capabilities"` + Count int `json:"count" jsonschema:"number of matching capabilities"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_find_capabilities", + Description: "Finds Edge capabilities matching the given criteria. All filter fields are ANDed; map/list field values are ORed. Useful for finding capabilities by edge site or parameter value.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUIDOptional("edgeSiteId", input.EdgeSiteId); err != nil { + return Output{}, err + } + caps, err := clients.FindCapabilities(ctx, collibraClient, clients.CapabilityFindRequest{ + EdgeSiteId: input.EdgeSiteId, + Labels: input.Labels, + Parameters: input.Parameters, + }) + if err != nil { + return Output{Capabilities: []clients.EdgeCapability{}, Error: fmt.Sprintf("failed to find capabilities: %s", err.Error())}, nil + } + if caps == nil { + caps = []clients.EdgeCapability{} + } + return Output{Capabilities: caps, Count: len(caps)}, nil + } +} \ No newline at end of file diff --git a/pkg/tools/edge_find_capabilities/tool_test.go b/pkg/tools/edge_find_capabilities/tool_test.go new file mode 100644 index 0000000..015b35a --- /dev/null +++ b/pkg/tools/edge_find_capabilities/tool_test.go @@ -0,0 +1,66 @@ +package edge_find_capabilities_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_find_capabilities" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeFindCapabilities(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/find", testutil.JsonHandlerInOut(func(r *http.Request, in clients.CapabilityFindRequest) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "cap-1", Name: "Databricks Cap", EdgeSiteId: in.EdgeSiteId}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + EdgeSiteId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + if len(output.Capabilities) != 1 || output.Capabilities[0].Id != "cap-1" { + t.Fatalf("expected cap-1, got %+v", output.Capabilities) + } +} + +func TestEdgeFindCapabilitiesAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/find", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "error" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error") + } +} + +func TestEdgeFindCapabilitiesInvalidSiteId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + EdgeSiteId: "not-a-uuid", + }) + if err == nil { + t.Fatal("expected validation error") + } +} \ No newline at end of file diff --git a/pkg/tools/edge_get_capability/tool.go b/pkg/tools/edge_get_capability/tool.go new file mode 100644 index 0000000..5b58f02 --- /dev/null +++ b/pkg/tools/edge_get_capability/tool.go @@ -0,0 +1,45 @@ +package edge_get_capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the capability to retrieve"` +} + +type Output struct { + Capability *clients.EdgeCapability `json:"capability,omitempty" jsonschema:"the capability details"` + Found bool `json:"found" jsonschema:"whether the capability was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_get_capability", + Description: "Gets an Edge capability by its UUID. Returns the capability's configuration, type, edge site, and parameters.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + cap, err := clients.GetCapability(ctx, collibraClient, input.Id) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get capability: %s", err.Error())}, nil + } + return Output{Capability: cap, Found: true}, nil + } +} \ No newline at end of file diff --git a/pkg/tools/edge_get_capability/tool_test.go b/pkg/tools/edge_get_capability/tool_test.go new file mode 100644 index 0000000..88590ff --- /dev/null +++ b/pkg/tools/edge_get_capability/tool_test.go @@ -0,0 +1,67 @@ +package edge_get_capability_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_get_capability" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeGetCapability(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.EdgeCapability) { + return http.StatusOK, clients.EdgeCapability{Id: "00000000-0000-0000-0000-000000000001", Name: "Test Cap"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Capability == nil || output.Capability.Name != "Test Cap" { + t.Fatal("expected capability name Test Cap") + } +} + +func TestEdgeGetCapabilityNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000002", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } + if output.Error == "" { + t.Fatal("expected error message") + } +} + +func TestEdgeGetCapabilityInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad-uuid"}) + if err == nil { + t.Fatal("expected validation error") + } +} \ No newline at end of file diff --git a/pkg/tools/edge_get_connection/tool.go b/pkg/tools/edge_get_connection/tool.go new file mode 100644 index 0000000..0b660e8 --- /dev/null +++ b/pkg/tools/edge_get_connection/tool.go @@ -0,0 +1,45 @@ +package edge_get_connection + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + ConnectionId string `json:"connectionId" jsonschema:"the UUID of the connection to retrieve"` +} + +type Output struct { + Connection *clients.EdgeConnection `json:"connection,omitempty" jsonschema:"the connection details"` + Found bool `json:"found" jsonschema:"whether the connection was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_get_connection", + Description: "Gets an Edge connection by its UUID. Returns the connection's configuration, type, edge site, and parameters.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("connectionId", input.ConnectionId); err != nil { + return Output{}, err + } + conn, err := clients.GetConnection(ctx, collibraClient, input.ConnectionId) + if err != nil { + return Output{Found: false, Error: fmt.Sprintf("failed to get connection: %s", err.Error())}, nil + } + return Output{Connection: conn, Found: true}, nil + } +} diff --git a/pkg/tools/edge_get_connection/tool_test.go b/pkg/tools/edge_get_connection/tool_test.go new file mode 100644 index 0000000..cf0ebc7 --- /dev/null +++ b/pkg/tools/edge_get_connection/tool_test.go @@ -0,0 +1,64 @@ +package edge_get_connection_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_get_connection" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeGetConnection(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections/00000000-0000-0000-0000-000000000001", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.EdgeConnection) { + return http.StatusOK, clients.EdgeConnection{Id: "00000000-0000-0000-0000-000000000001", Name: "Test Conn"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + ConnectionId: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Found { + t.Fatalf("expected found, got error: %s", output.Error) + } + if output.Connection.Name != "Test Conn" { + t.Fatalf("expected name Test Conn, got %s", output.Connection.Name) + } +} + +func TestEdgeGetConnectionNotFound(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections/00000000-0000-0000-0000-000000000002", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + ConnectionId: "00000000-0000-0000-0000-000000000002", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Found { + t.Fatal("expected not found") + } +} + +func TestEdgeGetConnectionInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ConnectionId: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_get_job_status_history/tool.go b/pkg/tools/edge_get_job_status_history/tool.go new file mode 100644 index 0000000..ab6cca7 --- /dev/null +++ b/pkg/tools/edge_get_job_status_history/tool.go @@ -0,0 +1,48 @@ +package edge_get_job_status_history + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the Edge job to get status history for"` +} + +type Output struct { + History []clients.JobStatusLog `json:"history" jsonschema:"all job status updates in LIFO order (most recent first)"` + Count int `json:"count" jsonschema:"number of status entries returned"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_get_job_status_history", + Description: "Gets all status updates for an Edge job in reverse chronological order (most recent first). Useful for diagnosing job failures.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + history, err := clients.GetEdgeJobStatusHistory(ctx, collibraClient, input.Id) + if err != nil { + return Output{History: []clients.JobStatusLog{}, Error: fmt.Sprintf("failed to get job status history: %s", err.Error())}, nil + } + if history == nil { + history = []clients.JobStatusLog{} + } + return Output{History: history, Count: len(history)}, nil + } +} diff --git a/pkg/tools/edge_get_job_status_history/tool_test.go b/pkg/tools/edge_get_job_status_history/tool_test.go new file mode 100644 index 0000000..ca4bae8 --- /dev/null +++ b/pkg/tools/edge_get_job_status_history/tool_test.go @@ -0,0 +1,70 @@ +package edge_get_job_status_history_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_get_job_status_history" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeGetJobStatusHistory(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/jobs/00000000-0000-0000-0000-000000000001/statusLogHistory", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.JobStatusLog) { + return http.StatusOK, []clients.JobStatusLog{ + {Status: "SUCCEEDED", Message: "Done"}, + {Status: "RUNNING", Message: "In progress"}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + if len(output.History) != 2 { + t.Fatalf("expected 2 entries, got %d", len(output.History)) + } + if output.History[0].Status != "SUCCEEDED" { + t.Fatalf("expected SUCCEEDED first, got %s", output.History[0].Status) + } +} + +func TestEdgeGetJobStatusHistoryAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/jobs/00000000-0000-0000-0000-000000000001/statusLogHistory", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error") + } +} + +func TestEdgeGetJobStatusHistoryInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/edge_list_capabilities/tool.go b/pkg/tools/edge_list_capabilities/tool.go new file mode 100644 index 0000000..71b60a0 --- /dev/null +++ b/pkg/tools/edge_list_capabilities/tool.go @@ -0,0 +1,74 @@ +package edge_list_capabilities + +import ( + "context" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + NameContains string `json:"nameContains,omitempty" jsonschema:"optional substring to filter capability names (case-insensitive)"` + TypeId string `json:"typeId,omitempty" jsonschema:"optional capability type ID to filter by, e.g. databricks-edge-capability"` + Limit int `json:"limit,omitempty" jsonschema:"optional maximum number of results to return"` +} + +type Output struct { + Capabilities []clients.EdgeCapability `json:"capabilities" jsonschema:"list of Edge capabilities"` + Count int `json:"count" jsonschema:"number of capabilities returned"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_list_capabilities", + Description: "Gets Edge capabilities with optional filtering by name (substring) and/or type ID. Use nameContains and typeId to avoid fetching the full list, which can be very large.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + caps, err := clients.ListCapabilities(ctx, collibraClient) + if err != nil { + return Output{Capabilities: []clients.EdgeCapability{}, Error: err.Error()}, nil + } + if caps == nil { + caps = []clients.EdgeCapability{} + } + + filtered := filter(caps, input) + return Output{Capabilities: filtered, Count: len(filtered)}, nil + } +} + +func filter(caps []clients.EdgeCapability, input Input) []clients.EdgeCapability { + result := make([]clients.EdgeCapability, 0, len(caps)) + nameFilter := strings.ToLower(input.NameContains) + typeFilter := strings.ToLower(input.TypeId) + + for _, c := range caps { + if nameFilter != "" && !strings.Contains(strings.ToLower(c.Name), nameFilter) { + continue + } + if typeFilter != "" { + typeId := "" + if c.Type != nil { + typeId = strings.ToLower(c.Type.Id) + } + if typeId != typeFilter { + continue + } + } + result = append(result, c) + if input.Limit > 0 && len(result) >= input.Limit { + break + } + } + return result +} diff --git a/pkg/tools/edge_list_capabilities/tool_test.go b/pkg/tools/edge_list_capabilities/tool_test.go new file mode 100644 index 0000000..668b19f --- /dev/null +++ b/pkg/tools/edge_list_capabilities/tool_test.go @@ -0,0 +1,144 @@ +package edge_list_capabilities_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_list_capabilities" + "github.com/collibra/chip/pkg/tools/testutil" +) + +var testCapabilities = []clients.EdgeCapability{ + {Id: "cap-1", Name: "vb-databricks-prod", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "cap-2", Name: "vb-databricks-staging", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "cap-3", Name: "finance-dataplex", Type: &clients.EdgeCapabilityType{Id: "dataplex-edge-capability"}}, + {Id: "cap-4", Name: "sales-databricks", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, +} + +func newServer(t *testing.T) *httptest.Server { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, testCapabilities + })) + return httptest.NewServer(handler) +} + +func TestEdgeListCapabilities(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + if output.Count != 4 { + t.Fatalf("expected 4 capabilities, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesFilterByName(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{NameContains: "vb"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 2 { + t.Fatalf("expected 2 capabilities, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesFilterByType(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{TypeId: "databricks-edge-capability"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 3 { + t.Fatalf("expected 3 databricks capabilities, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesFilterByNameAndType(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + NameContains: "vb", + TypeId: "databricks-edge-capability", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 2 { + t.Fatalf("expected 2 capabilities, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesFilterByNameCaseInsensitive(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{NameContains: "VB"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 2 { + t.Fatalf("expected 2 capabilities (case-insensitive), got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesLimit(t *testing.T) { + server := newServer(t) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Limit: 2}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 2 { + t.Fatalf("expected 2 capabilities with limit, got %d", output.Count) + } +} + +func TestEdgeListCapabilitiesAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "internal server error" + })) + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error to be set") + } +} + +func TestEdgeListCapabilitiesEmpty(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{} + })) + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 0 { + t.Fatalf("expected count 0, got %d", output.Count) + } +} diff --git a/pkg/tools/edge_list_connections/tool.go b/pkg/tools/edge_list_connections/tool.go new file mode 100644 index 0000000..01a95cb --- /dev/null +++ b/pkg/tools/edge_list_connections/tool.go @@ -0,0 +1,41 @@ +package edge_list_connections + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct{} + +type Output struct { + Connections []clients.EdgeConnection `json:"connections" jsonschema:"list of Edge connections"` + Count int `json:"count" jsonschema:"number of connections returned"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_list_connections", + Description: "Gets all available Edge connections. Returns the full list of connections configured on the Collibra Edge management service.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + conns, err := clients.ListConnections(ctx, collibraClient) + if err != nil { + return Output{Connections: []clients.EdgeConnection{}, Error: err.Error()}, nil + } + if conns == nil { + conns = []clients.EdgeConnection{} + } + return Output{Connections: conns, Count: len(conns)}, nil + } +} diff --git a/pkg/tools/edge_list_connections/tool_test.go b/pkg/tools/edge_list_connections/tool_test.go new file mode 100644 index 0000000..3ffc69d --- /dev/null +++ b/pkg/tools/edge_list_connections/tool_test.go @@ -0,0 +1,55 @@ +package edge_list_connections_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_list_connections" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeListConnections(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeConnection) { + return http.StatusOK, []clients.EdgeConnection{ + {Id: "conn-1", Name: "Databricks Conn"}, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + if len(output.Connections) != 1 || output.Connections[0].Id != "conn-1" { + t.Fatalf("expected conn-1, got %+v", output.Connections) + } + if output.Count != 1 { + t.Fatalf("expected count 1, got %d", output.Count) + } +} + +func TestEdgeListConnectionsAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/connections", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "error" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error") + } +} diff --git a/pkg/tools/edge_run_capability/tool.go b/pkg/tools/edge_run_capability/tool.go new file mode 100644 index 0000000..2c40162 --- /dev/null +++ b/pkg/tools/edge_run_capability/tool.go @@ -0,0 +1,57 @@ +package edge_run_capability + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Id string `json:"id" jsonschema:"the UUID of the capability to run"` + JobId string `json:"jobId,omitempty" jsonschema:"optional UUID for tracking the job execution"` + InFastNamespace bool `json:"inFastNamespace,omitempty" jsonschema:"if true, runs in the fast (priority) namespace"` + Parameters map[string]any `json:"parameters,omitempty" jsonschema:"optional run-time parameters for this execution only"` + WorkflowName string `json:"workflowName,omitempty" jsonschema:"optional workflow name; defaults to workflow.yaml inside the capability package"` +} + +type Output struct { + JobId string `json:"jobId,omitempty" jsonschema:"UUID of the job submitted to Edge"` + Success bool `json:"success" jsonschema:"whether the run was submitted successfully"` + Error string `json:"error,omitempty" jsonschema:"error message if submission failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_run_capability", + Description: "Runs an Edge capability immediately. Returns the job UUID that can be used to track execution via edge_get_job_status.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("id", input.Id); err != nil { + return Output{}, err + } + if err := validation.UUIDOptional("jobId", input.JobId); err != nil { + return Output{}, err + } + jobId, err := clients.RunCapability(ctx, collibraClient, input.Id, clients.CapabilityRunRequest{ + JobId: input.JobId, + InFastNamespace: input.InFastNamespace, + Parameters: input.Parameters, + WorkflowName: input.WorkflowName, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to run capability: %s", err.Error())}, nil + } + return Output{JobId: jobId, Success: true}, nil + } +} diff --git a/pkg/tools/edge_run_capability/tool_test.go b/pkg/tools/edge_run_capability/tool_test.go new file mode 100644 index 0000000..5e94f8b --- /dev/null +++ b/pkg/tools/edge_run_capability/tool_test.go @@ -0,0 +1,66 @@ +package edge_run_capability_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + tools "github.com/collibra/chip/pkg/tools/edge_run_capability" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestEdgeRunCapability(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000001/run", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusCreated, "00000000-0000-0000-0000-000000000099" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got: %s", output.Error) + } + if output.JobId != "00000000-0000-0000-0000-000000000099" { + t.Fatalf("expected job id 00000000-0000-0000-0000-000000000099, got: %s", output.JobId) + } +} + +func TestEdgeRunCapabilityAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities/00000000-0000-0000-0000-000000000001/run", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "capability not found" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Id: "00000000-0000-0000-0000-000000000001", + }) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Success { + t.Fatal("expected failure") + } + if output.Error == "" { + t.Fatal("expected error message") + } +} + +func TestEdgeRunCapabilityInvalidId(t *testing.T) { + server := httptest.NewServer(http.NewServeMux()) + defer server.Close() + + _, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Id: "bad"}) + if err == nil { + t.Fatal("expected validation error") + } +} diff --git a/pkg/tools/find_connections/tool.go b/pkg/tools/find_connections/tool.go index 605ad77..60291a1 100644 --- a/pkg/tools/find_connections/tool.go +++ b/pkg/tools/find_connections/tool.go @@ -22,7 +22,7 @@ type Input struct { } type Output struct { - Connections []clients.Connection `json:"connections" jsonschema:"The matching connections."` + Connections []clients.EdgeConnection `json:"connections" jsonschema:"The matching connections."` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { diff --git a/pkg/tools/find_connections/tool_test.go b/pkg/tools/find_connections/tool_test.go index 8deef26..6706aad 100644 --- a/pkg/tools/find_connections/tool_test.go +++ b/pkg/tools/find_connections/tool_test.go @@ -16,12 +16,12 @@ func TestFindConnections(t *testing.T) { siteID, _ := uuid.NewUUID() handler := http.NewServeMux() - handler.Handle("POST /edge/api/rest/v2/connections/find", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionFindRequest) (int, []clients.Connection) { + handler.Handle("POST /edge/api/rest/v2/connections/find", testutil.JsonHandlerInOut(func(r *http.Request, in clients.ConnectionFindRequest) (int, []clients.EdgeConnection) { if in.Name != "snowflake-manual" { t.Fatalf("unexpected name: %s", in.Name) } - return http.StatusOK, []clients.Connection{ - {ID: connID.String(), Name: "snowflake-manual", EdgeSiteID: siteID.String()}, + return http.StatusOK, []clients.EdgeConnection{ + {Id: connID.String(), Name: "snowflake-manual", EdgeSiteId: siteID.String()}, } })) @@ -33,7 +33,7 @@ func TestFindConnections(t *testing.T) { if err != nil { t.Fatalf("expected no error, got: %v", err) } - if len(output.Connections) != 1 || output.Connections[0].ID != connID.String() { + if len(output.Connections) != 1 || output.Connections[0].Id != connID.String() { t.Fatalf("unexpected output: %+v", output) } } diff --git a/pkg/tools/get_catalog_job_status/tool.go b/pkg/tools/get_catalog_job_status/tool.go index db4636a..6b2e35f 100644 --- a/pkg/tools/get_catalog_job_status/tool.go +++ b/pkg/tools/get_catalog_job_status/tool.go @@ -19,14 +19,14 @@ type Input struct { } type Output struct { - Name string `json:"name,omitempty" jsonschema:"The job's name."` - Type string `json:"type,omitempty" jsonschema:"The job's type (e.g. 'DELTA_INGESTION')."` - State string `json:"state,omitempty" jsonschema:"The job's current state: WAITING, RUNNING, CANCELING, COMPLETED, CANCELED, or ERROR."` - Result string `json:"result,omitempty" jsonschema:"The job's result once finished: NOT_SET (not finished yet), SUCCESS, COMPLETED_WITH_ERROR, FAILURE, or ABORTED."` - Message string `json:"message,omitempty" jsonschema:"Status message from the job."` - ProgressPercentage float64 `json:"progressPercentage,omitempty" jsonschema:"Progress percentage, 0-100."` - StartDate string `json:"startDate,omitempty"` - EndDate string `json:"endDate,omitempty" jsonschema:"When the job finished, if it has."` + Name string `json:"name,omitempty" jsonschema:"The job's name."` + Type string `json:"type,omitempty" jsonschema:"The job's type (e.g. 'DELTA_INGESTION')."` + State string `json:"state,omitempty" jsonschema:"The job's current state: WAITING, RUNNING, CANCELING, COMPLETED, CANCELED, or ERROR."` + Result string `json:"result,omitempty" jsonschema:"The job's result once finished: NOT_SET (not finished yet), SUCCESS, COMPLETED_WITH_ERROR, FAILURE, or ABORTED."` + Message string `json:"message,omitempty" jsonschema:"Status message from the job."` + ProgressPercentage int `json:"progressPercentage,omitempty" jsonschema:"Progress percentage, 0-100."` + StartDate string `json:"startDate,omitempty"` + EndDate string `json:"endDate,omitempty" jsonschema:"When the job finished, if it has."` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { @@ -46,7 +46,7 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { return Output{}, err } - job, err := clients.GetCatalogJob(ctx, collibraClient, input.JobID) + job, err := clients.GetJob(ctx, collibraClient, input.JobID) if err != nil { return Output{}, err } diff --git a/pkg/tools/jobs_find/tool.go b/pkg/tools/jobs_find/tool.go new file mode 100644 index 0000000..d6e3586 --- /dev/null +++ b/pkg/tools/jobs_find/tool.go @@ -0,0 +1,51 @@ +package jobs_find + +import ( + "context" + "fmt" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Name string `json:"name,omitempty" jsonschema:"filter by job name"` + NameMatchMode string `json:"nameMatchMode,omitempty" jsonschema:"name match mode: EXACT or ANYWHERE"` + Result []string `json:"result,omitempty" jsonschema:"filter by result: NOT_SET, SUCCESS, COMPLETED_WITH_ERROR, FAILURE, ABORTED"` + State []string `json:"state,omitempty" jsonschema:"filter by state: WAITING, RUNNING, COMPLETED, FAILED, DELETED"` + Type []string `json:"type,omitempty" jsonschema:"filter by job type"` + User string `json:"user,omitempty" jsonschema:"filter by user who created the job"` + SortField string `json:"sortField,omitempty" jsonschema:"field to sort by"` + SortOrder string `json:"sortOrder,omitempty" jsonschema:"sort order: ASC or DESC"` + Cursor string `json:"cursor,omitempty" jsonschema:"pagination cursor from previous response"` + PageSize int `json:"pageSize,omitempty" jsonschema:"number of results per page"` +} + +type Output struct { + Jobs []clients.Job `json:"jobs,omitempty" jsonschema:"list of matching jobs"` + Count int `json:"count" jsonschema:"number of jobs returned"` + NextCursor string `json:"nextCursor,omitempty" jsonschema:"cursor for the next page of results"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "jobs_find", + Description: "Finds jobs matching the given filter criteria in the Collibra Jobs API.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + resp, err := clients.FindJobs(ctx, collibraClient, input.Name, input.NameMatchMode, input.Result, input.State, input.Type, input.User, input.SortField, input.SortOrder, input.Cursor, input.PageSize) + if err != nil { + return Output{Error: fmt.Sprintf("failed to find jobs: %s", err.Error())}, nil + } + return Output{Jobs: resp.Results, Count: len(resp.Results), NextCursor: resp.NextCursor}, nil + } +} diff --git a/pkg/tools/jobs_find/tool_test.go b/pkg/tools/jobs_find/tool_test.go new file mode 100644 index 0000000..aa50fbc --- /dev/null +++ b/pkg/tools/jobs_find/tool_test.go @@ -0,0 +1,74 @@ +package jobs_find_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/jobs_find" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestJobsFind(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobPagedResponse) { + return http.StatusOK, clients.JobPagedResponse{ + Results: []clients.Job{ + {ID: "job-1", Name: "test-job", State: "COMPLETED"}, + }, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{ + Name: "test-job", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("expected no error, got: %s", output.Error) + } + if output.Count != 1 || output.Jobs[0].ID != "job-1" { + t.Fatal("expected one job with id job-1") + } +} + +func TestJobsFindAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "internal error" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected error message in output") + } +} + +func TestJobsFindNoFilters(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobPagedResponse) { + return http.StatusOK, clients.JobPagedResponse{Results: []clients.Job{}} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Count != 0 { + t.Fatalf("expected 0 jobs, got %d", output.Count) + } +} diff --git a/pkg/tools/list_integrations/tool.go b/pkg/tools/list_integrations/tool.go new file mode 100644 index 0000000..2a1c17b --- /dev/null +++ b/pkg/tools/list_integrations/tool.go @@ -0,0 +1,205 @@ +package list_integrations + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + Platform string `json:"platform,omitempty" jsonschema:"optional platform filter: databricks, dataplex, etc. Matches type.Id substring. If omitted, all capabilities are returned."` + NameContains string `json:"nameContains,omitempty" jsonschema:"optional case-insensitive substring filter on capability name"` +} + +type Integration struct { + IngestibleId string `json:"ingestibleId" jsonschema:"UUID of the integration instance, use this for all subsequent calls"` + Name string `json:"name" jsonschema:"capability name"` + TypeId string `json:"typeId" jsonschema:"capability type, e.g. databricks-edge-capability"` + EdgeSiteId string `json:"edgeSiteId" jsonschema:"Edge site this capability runs on"` + HasSchedule bool `json:"hasSchedule" jsonschema:"whether a sync schedule is configured"` + CronExpression string `json:"cronExpression,omitempty" jsonschema:"cron expression for the schedule"` + CronTimeZone string `json:"cronTimeZone,omitempty" jsonschema:"timezone for the cron schedule"` + LastRun string `json:"lastRun,omitempty" jsonschema:"relative time of last sync run, e.g. 6h ago"` + LastRunAt string `json:"lastRunAt,omitempty" jsonschema:"ISO 8601 timestamp of last sync run, use this for precise time comparisons e.g. ran in last 24h"` + LastRunState string `json:"lastRunState,omitempty" jsonschema:"state of last job: COMPLETED, FAILED, RUNNING, etc."` + LastRunResult string `json:"lastRunResult,omitempty" jsonschema:"result of last job: SUCCESS, FAILURE, COMPLETED_WITH_ERROR, etc."` + NextRun string `json:"nextRun,omitempty" jsonschema:"relative time of next scheduled run, e.g. in 18h"` +} + +type Output struct { + Integrations []Integration `json:"integrations" jsonschema:"list of integrations with schedule info"` + Total int `json:"total" jsonschema:"total number of integrations returned"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "list_integrations", + Description: `Lists all integration capabilities with their sync schedule and last/next run times. +Use this as the entry point for any integration question: "show me all integrations", "show Databricks integrations", "what's scheduled for the Finance integration?". +Optionally filter by platform substring (e.g. "databricks", "dataplex") and/or name. Returns ingestibleId for each — pass it to catalog_generic_get_schedule, catalog_generic_start_job, or catalog_generic_cancel_job. +For time-based queries like "which integrations ran in the last 24h", use jobs_find sorted by startDate to get recent jobs, then cross-reference with integration names from this tool.`, + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + caps, err := clients.ListCapabilities(ctx, collibraClient) + if err != nil { + return Output{Error: fmt.Sprintf("failed to list capabilities: %s", err.Error())}, nil + } + + filtered := filterCapabilities(caps, input) + integrations := enrichWithSchedules(ctx, collibraClient, filtered) + return Output{Integrations: integrations, Total: len(integrations)}, nil + } +} + +func filterCapabilities(caps []clients.EdgeCapability, input Input) []clients.EdgeCapability { + platform := strings.ToLower(input.Platform) + name := strings.ToLower(input.NameContains) + + result := make([]clients.EdgeCapability, 0, len(caps)) + for _, c := range caps { + if platform != "" && platform != "all" { + typeId := "" + if c.Type != nil { + typeId = strings.ToLower(c.Type.Id) + } + if !strings.Contains(typeId, platform) { + continue + } + } + + if name != "" && !strings.Contains(strings.ToLower(c.Name), name) { + continue + } + + result = append(result, c) + } + return result +} + +func enrichWithSchedules(ctx context.Context, collibraClient *http.Client, caps []clients.EdgeCapability) []Integration { + integrations := make([]Integration, len(caps)) + now := time.Now() + + var wg sync.WaitGroup + for i, c := range caps { + wg.Add(1) + go func(idx int, cap clients.EdgeCapability) { + defer wg.Done() + integrations[idx] = buildIntegration(ctx, collibraClient, cap, now) + }(i, c) + } + wg.Wait() + return integrations +} + +func buildIntegration(ctx context.Context, collibraClient *http.Client, cap clients.EdgeCapability, now time.Time) Integration { + typeId := "" + if cap.Type != nil { + typeId = cap.Type.Id + } + + integration := Integration{ + IngestibleId: cap.Id, + Name: cap.Name, + TypeId: typeId, + EdgeSiteId: cap.EdgeSiteId, + } + + schedule, err := clients.GetGenericSchedule(ctx, collibraClient, cap.Id) + if err == nil && schedule != nil { + integration.HasSchedule = true + integration.CronExpression = schedule.CronExpression + integration.CronTimeZone = schedule.CronTimeZone + integration.NextRun = futureRelativeTime(schedule.NextRunDateLongValue, now) + if schedule.LastRunTimeStamp > 0 { + t := time.UnixMilli(schedule.LastRunTimeStamp) + integration.LastRun = relativeTime(t, now) + integration.LastRunAt = t.UTC().Format(time.RFC3339) + } + } + + // fall back to Jobs API for last run when schedule has no timestamp + if integration.LastRun == "" { + fillLastRunFromJobs(ctx, collibraClient, &integration, cap.Name, now) + } + + return integration +} + +func fillLastRunFromJobs(ctx context.Context, collibraClient *http.Client, integration *Integration, capName string, now time.Time) { + resp, err := clients.FindJobs(ctx, collibraClient, capName, "ANYWHERE", nil, nil, nil, "", "START_DATE", "DESC", "", 1) + if err != nil || len(resp.Results) == 0 { + return + } + job := resp.Results[0] + integration.LastRunState = job.State + integration.LastRunResult = job.Result + + // prefer EndDate for completed jobs, fall back to StartDate + dateStr := job.EndDate + if dateStr == "" { + dateStr = job.StartDate + } + if t := parseJobDate(dateStr); !t.IsZero() { + integration.LastRun = relativeTime(t, now) + integration.LastRunAt = t.UTC().Format(time.RFC3339) + } +} + +func parseJobDate(s string) time.Time { + if s == "" { + return time.Time{} + } + for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02T15:04:05.999Z07:00"} { + if t, err := time.Parse(layout, s); err == nil { + return t + } + } + return time.Time{} +} + +func relativeTime(t time.Time, now time.Time) string { + diff := now.Sub(t) + switch { + case diff < time.Minute: + return "just now" + case diff < time.Hour: + return fmt.Sprintf("%dm ago", int(diff.Minutes())) + case diff < 24*time.Hour: + return fmt.Sprintf("%dh ago", int(diff.Hours())) + default: + return fmt.Sprintf("%dd ago", int(diff.Hours()/24)) + } +} + +func futureRelativeTime(epochMs int64, now time.Time) string { + if epochMs == 0 { + return "" + } + diff := time.UnixMilli(epochMs).Sub(now) + if diff <= 0 { + return "overdue" + } + switch { + case diff < time.Hour: + return fmt.Sprintf("in %dm", int(diff.Minutes())) + case diff < 24*time.Hour: + return fmt.Sprintf("in %dh", int(diff.Hours())) + default: + return fmt.Sprintf("in %dd", int(diff.Hours()/24)) + } +} diff --git a/pkg/tools/list_integrations/tool_test.go b/pkg/tools/list_integrations/tool_test.go new file mode 100644 index 0000000..40ebda5 --- /dev/null +++ b/pkg/tools/list_integrations/tool_test.go @@ -0,0 +1,217 @@ +package list_integrations_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/list_integrations" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func TestListIntegrations(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Sales", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "00000000-0000-0000-0000-000000000002", Name: "UC / Finance", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "00000000-0000-0000-0000-000000000003", Name: "Dataplex Finance", Type: &clients.EdgeCapabilityType{Id: "dataplex-synchronization"}}, + {Id: "00000000-0000-0000-0000-000000000004", Name: "jdbc-only", Type: &clients.EdgeCapabilityType{Id: "jdbc-profiling"}}, + } + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{CronExpression: "0 2 * * *", CronTimeZone: "UTC", LastRunTimeStamp: 1000, NextRunDateLongValue: 9999999999000} + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000002/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{CronExpression: "0 3 * * *", CronTimeZone: "UTC"} + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000003/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "no schedule" + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000004/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "no schedule" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Error != "" { + t.Fatalf("unexpected output error: %s", output.Error) + } + // all capabilities returned when no platform filter + if output.Total != 4 { + t.Fatalf("expected 4 integrations (all types), got %d", output.Total) + } +} + +func TestListIntegrationsFilterByPlatform(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Sales", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + {Id: "00000000-0000-0000-0000-000000000002", Name: "Dataplex Finance", Type: &clients.EdgeCapabilityType{Id: "dataplex-synchronization"}}, + } + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{CronExpression: "0 2 * * *"} + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{Platform: "databricks"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Total != 1 { + t.Fatalf("expected 1 databricks integration, got %d", output.Total) + } + if output.Integrations[0].Name != "UC / Sales" { + t.Fatalf("expected UC / Sales, got %s", output.Integrations[0].Name) + } +} + +func TestListIntegrationsScheduleEnrichment(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Sales", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + } + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{ + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + LastRunTimeStamp: 1000, + NextRunDateLongValue: 9999999999000, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Total != 1 { + t.Fatalf("expected 1 integration, got %d", output.Total) + } + integration := output.Integrations[0] + if !integration.HasSchedule { + t.Fatal("expected HasSchedule to be true") + } + if integration.CronExpression != "0 2 * * *" { + t.Fatalf("expected cron 0 2 * * *, got %s", integration.CronExpression) + } + if integration.LastRun == "" { + t.Fatal("expected LastRun to be set") + } + if integration.NextRun == "" { + t.Fatal("expected NextRun to be set") + } +} + +func TestListIntegrationsNoScheduleFallsBackToJobs(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Marketing", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + } + })) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusNotFound, "not found" + })) + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobPagedResponse) { + return http.StatusOK, clients.JobPagedResponse{ + Results: []clients.Job{ + {ID: "job-1", Name: "UC / Marketing", State: "COMPLETED", Result: "SUCCESS", EndDate: "2026-05-01T02:00:00Z"}, + }, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if output.Total != 1 { + t.Fatalf("expected 1 integration, got %d", output.Total) + } + integration := output.Integrations[0] + if integration.HasSchedule { + t.Fatal("expected HasSchedule to be false") + } + if integration.LastRun == "" { + t.Fatal("expected LastRun to be populated from jobs API") + } + if integration.LastRunState != "COMPLETED" { + t.Fatalf("expected LastRunState COMPLETED, got %s", integration.LastRunState) + } + if integration.LastRunResult != "SUCCESS" { + t.Fatalf("expected LastRunResult SUCCESS, got %s", integration.LastRunResult) + } +} + +func TestListIntegrationsScheduleNoLastRunFallsBackToJobs(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, []clients.EdgeCapability) { + return http.StatusOK, []clients.EdgeCapability{ + {Id: "00000000-0000-0000-0000-000000000001", Name: "UC / Sales", Type: &clients.EdgeCapabilityType{Id: "databricks-edge-capability"}}, + } + })) + // schedule exists but lastRunTimeStamp is 0 (never run) + handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { + return http.StatusOK, clients.GenericSchedule{CronExpression: "0 2 * * *", CronTimeZone: "UTC", LastRunTimeStamp: 0} + })) + handler.Handle("/rest/jobs/v1/jobs", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobPagedResponse) { + return http.StatusOK, clients.JobPagedResponse{ + Results: []clients.Job{ + {ID: "job-1", State: "FAILED", Result: "FAILURE", EndDate: "2026-04-30T03:00:00Z"}, + }, + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + integration := output.Integrations[0] + if !integration.HasSchedule { + t.Fatal("expected HasSchedule to be true") + } + if integration.LastRun == "" { + t.Fatal("expected LastRun from jobs fallback") + } + if integration.LastRunState != "FAILED" { + t.Fatalf("expected LastRunState FAILED, got %s", integration.LastRunState) + } +} + +func TestListIntegrationsAPIError(t *testing.T) { + handler := http.NewServeMux() + handler.Handle("/edge/api/rest/v2/capabilities", testutil.JsonHandlerOut(func(r *http.Request) (int, string) { + return http.StatusInternalServerError, "error" + })) + + server := httptest.NewServer(handler) + defer server.Close() + + output, err := tools.NewTool(testutil.NewClient(server)).Handler(t.Context(), tools.Input{}) + if err != nil { + t.Fatalf("unexpected go error: %v", err) + } + if output.Error == "" { + t.Fatal("expected output error to be set") + } +} diff --git a/pkg/tools/register.go b/pkg/tools/register.go index 52bb013..3512069 100644 --- a/pkg/tools/register.go +++ b/pkg/tools/register.go @@ -7,6 +7,17 @@ import ( "github.com/collibra/chip/pkg/chip" "github.com/collibra/chip/pkg/skills" "github.com/collibra/chip/pkg/tools/add_data_classification_match" + "github.com/collibra/chip/pkg/tools/catalog_generic_add_schedule" + "github.com/collibra/chip/pkg/tools/catalog_generic_cancel_job" + "github.com/collibra/chip/pkg/tools/catalog_generic_delete_config" + "github.com/collibra/chip/pkg/tools/catalog_generic_delete_schedule" + "github.com/collibra/chip/pkg/tools/catalog_generic_get_all_schedules" + "github.com/collibra/chip/pkg/tools/catalog_generic_get_config" + "github.com/collibra/chip/pkg/tools/catalog_generic_get_schedule" + "github.com/collibra/chip/pkg/tools/catalog_generic_get_schema" + "github.com/collibra/chip/pkg/tools/catalog_generic_save_config" + "github.com/collibra/chip/pkg/tools/catalog_generic_start_job" + "github.com/collibra/chip/pkg/tools/catalog_generic_update_schedule" "github.com/collibra/chip/pkg/tools/configure_database" "github.com/collibra/chip/pkg/tools/create_asset" "github.com/collibra/chip/pkg/tools/create_capability" @@ -15,6 +26,16 @@ import ( "github.com/collibra/chip/pkg/tools/create_domain" "github.com/collibra/chip/pkg/tools/discover_business_glossary" "github.com/collibra/chip/pkg/tools/discover_data_assets" + "github.com/collibra/chip/pkg/tools/edge_cancel_job" + "github.com/collibra/chip/pkg/tools/edge_delete_capability" + "github.com/collibra/chip/pkg/tools/edge_delete_connection" + "github.com/collibra/chip/pkg/tools/edge_find_capabilities" + "github.com/collibra/chip/pkg/tools/edge_get_capability" + "github.com/collibra/chip/pkg/tools/edge_get_connection" + "github.com/collibra/chip/pkg/tools/edge_get_job_status_history" + "github.com/collibra/chip/pkg/tools/edge_list_capabilities" + "github.com/collibra/chip/pkg/tools/edge_list_connections" + "github.com/collibra/chip/pkg/tools/edge_run_capability" "github.com/collibra/chip/pkg/tools/edit_asset" "github.com/collibra/chip/pkg/tools/find_connections" "github.com/collibra/chip/pkg/tools/find_domain_types" @@ -34,11 +55,13 @@ import ( "github.com/collibra/chip/pkg/tools/get_measure_data" "github.com/collibra/chip/pkg/tools/get_table_semantics" "github.com/collibra/chip/pkg/tools/init_data_contract" + "github.com/collibra/chip/pkg/tools/jobs_find" "github.com/collibra/chip/pkg/tools/list_asset_types" "github.com/collibra/chip/pkg/tools/list_capability_types" "github.com/collibra/chip/pkg/tools/list_context_specifications" "github.com/collibra/chip/pkg/tools/list_data_contracts" "github.com/collibra/chip/pkg/tools/list_edge_sites" + "github.com/collibra/chip/pkg/tools/list_integrations" "github.com/collibra/chip/pkg/tools/prepare_create_asset" "github.com/collibra/chip/pkg/tools/pull_data_contract_manifest" "github.com/collibra/chip/pkg/tools/push_data_contract_manifest" @@ -117,6 +140,36 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, configure_database.NewTool(client)) toolRegister(server, toolConfig, start_ingestion.NewTool(client)) + // Edge Management tools — read/delete/run operations with no counterpart among the + // create/find/test/get-status lifecycle tools above (create_connection and + // create_capability already upsert, so no edge update tools are needed here). + toolRegister(server, toolConfig, edge_list_capabilities.NewTool(client)) + toolRegister(server, toolConfig, edge_find_capabilities.NewTool(client)) + toolRegister(server, toolConfig, edge_get_capability.NewTool(client)) + toolRegister(server, toolConfig, edge_delete_capability.NewTool(client)) + toolRegister(server, toolConfig, edge_run_capability.NewTool(client)) + toolRegister(server, toolConfig, edge_list_connections.NewTool(client)) + toolRegister(server, toolConfig, edge_get_connection.NewTool(client)) + toolRegister(server, toolConfig, edge_delete_connection.NewTool(client)) + toolRegister(server, toolConfig, edge_cancel_job.NewTool(client)) + toolRegister(server, toolConfig, edge_get_job_status_history.NewTool(client)) + // Catalog Generic Integration tools + toolRegister(server, toolConfig, catalog_generic_get_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_save_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_delete_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_get_schema.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_get_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_add_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_update_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_delete_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_get_all_schedules.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_cancel_job.NewTool(client)) + toolRegister(server, toolConfig, catalog_generic_start_job.NewTool(client)) + // Jobs tools + toolRegister(server, toolConfig, jobs_find.NewTool(client)) + // Integration lifecycle tools + toolRegister(server, toolConfig, list_integrations.NewTool(client)) + if toolConfig.EnableDebugTools { toolRegister(server, toolConfig, get_debug_mcp_init_request.NewTool(client)) } diff --git a/pkg/tools/start_ingestion/tool.go b/pkg/tools/start_ingestion/tool.go index cab4a51..0b99178 100644 --- a/pkg/tools/start_ingestion/tool.go +++ b/pkg/tools/start_ingestion/tool.go @@ -19,7 +19,7 @@ type Input struct { } type Output struct { - Job *clients.Job `json:"job,omitempty" jsonschema:"The started synchronization job. Poll its id with get_catalog_job_status (NOT get_job_status, which is for Edge-site jobs) to see when it completes."` + Job *clients.CatalogJob `json:"job,omitempty" jsonschema:"The started synchronization job. Poll its id with get_catalog_job_status (NOT get_job_status, which is for Edge-site jobs) to see when it completes."` Success bool `json:"success" jsonschema:"Whether the ingestion job was started."` Error string `json:"error,omitempty" jsonschema:"Error message if starting the job failed."` } diff --git a/pkg/tools/start_ingestion/tool_test.go b/pkg/tools/start_ingestion/tool_test.go index 41467c9..1061576 100644 --- a/pkg/tools/start_ingestion/tool_test.go +++ b/pkg/tools/start_ingestion/tool_test.go @@ -17,11 +17,11 @@ func TestStartIngestion_Success(t *testing.T) { handler := http.NewServeMux() handler.Handle("POST /rest/catalogDatabase/v1/databases/"+databaseID.String()+"/synchronizeMetadata", - testutil.JsonHandlerInOut(func(r *http.Request, in clients.DatabaseMetadataSynchronizationRequest) (int, clients.Job) { + testutil.JsonHandlerInOut(func(r *http.Request, in clients.DatabaseMetadataSynchronizationRequest) (int, clients.CatalogJob) { if len(in.SchemaConnectionIDs) != 0 { t.Fatalf("expected no schemaConnectionIds, got: %v", in.SchemaConnectionIDs) } - return http.StatusAccepted, clients.Job{ID: jobID.String(), State: "RUNNING"} + return http.StatusAccepted, clients.CatalogJob{ID: jobID.String(), State: "RUNNING"} })) server := httptest.NewServer(handler) @@ -49,11 +49,11 @@ func TestStartIngestion_WithSchemaConnectionIDs(t *testing.T) { handler := http.NewServeMux() handler.Handle("POST /rest/catalogDatabase/v1/databases/"+databaseID.String()+"/synchronizeMetadata", - testutil.JsonHandlerInOut(func(r *http.Request, in clients.DatabaseMetadataSynchronizationRequest) (int, clients.Job) { + testutil.JsonHandlerInOut(func(r *http.Request, in clients.DatabaseMetadataSynchronizationRequest) (int, clients.CatalogJob) { if len(in.SchemaConnectionIDs) != 1 || in.SchemaConnectionIDs[0] != schemaConnID.String() { t.Fatalf("unexpected schemaConnectionIds: %v", in.SchemaConnectionIDs) } - return http.StatusAccepted, clients.Job{ID: jobID.String(), State: "RUNNING"} + return http.StatusAccepted, clients.CatalogJob{ID: jobID.String(), State: "RUNNING"} })) server := httptest.NewServer(handler) From 65cb326cb75046a0ddd5cdcb82d198554f5681c2 Mon Sep 17 00:00:00 2001 From: "volodymyr.bodnar" Date: Tue, 7 Jul 2026 20:21:35 -0400 Subject: [PATCH 08/10] feat: add etl-integration skill and rename integration tools Tool renames (clarity + consistent namespaces): - create_connection/create_capability -> edge_create_connection/edge_create_capability - find_connections -> edge_find_connections; list_edge_sites/list_capability_types -> edge_list_* - get_catalog_job_status -> get_job_status (DGC job); get_job_status -> edge_get_job_status (Edge job) - catalog_generic_* -> catalog_etl_*; save_config/delete_config -> save_generic_config/delete_generic_config - descriptions: "GENERIC integration instance" -> "ETL integration" Client consolidation: - per-entity Edge clients (edge_capability/edge_connection/edge_job) - canonical DGC Job type in dgc_job_client; catalog DB-sync response -> CatalogJob New skill collibra/etl-integration: - full lifecycle (edge connection/capability -> catalog_etl generic config -> schedule/run) - runtime schema introspection via catalog_etl_get_schema (no embedded schemas) - DGC prerequisites (domain/community/System) with create-or-reuse prompt - ingestion-path disambiguation vs jdbc-ingestion - 15 per-integration references with linked Collibra product docs Indexes updated: README, SKILLS.md, collibra/index navigator. Ignore .DS_Store. --- .gitignore | 1 + README.md | 14 +- SKILLS.md | 1 + pkg/clients/data_source_docs_client.go | 2 +- pkg/clients/dgc_job_client.go | 2 +- pkg/clients/generic_integration_client.go | 44 ++--- pkg/clients/prepare_create_asset_client.go | 17 +- .../files/collibra/etl-integration/SKILL.md | 172 ++++++++++++++++++ .../references/azure-foundry.md | 67 +++++++ .../etl-integration/references/azureml.md | 24 +++ .../etl-integration/references/bedrock.md | 42 +++++ .../references/databricks-lineage.md | 32 ++++ .../references/databricks-uc.md | 47 +++++ .../references/dataplex-lineage.md | 32 ++++ .../etl-integration/references/dataplex.md | 36 ++++ .../etl-integration/references/datazone.md | 27 +++ .../etl-integration/references/fabric.md | 57 ++++++ .../etl-integration/references/index.md | 40 ++++ .../etl-integration/references/mlflow.md | 24 +++ .../etl-integration/references/purview.md | 37 ++++ .../etl-integration/references/sagemaker.md | 26 +++ .../etl-integration/references/sigma.md | 23 +++ .../etl-integration/references/thoughtspot.md | 37 ++++ .../etl-integration/references/vertex.md | 38 ++++ pkg/skills/files/collibra/index/SKILL.md | 3 +- .../files/collibra/jdbc-ingestion/SKILL.md | 41 +++-- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- .../tool.go | 8 +- .../tool_test.go | 4 +- pkg/tools/configure_database/tool.go | 8 +- .../tool.go | 16 +- .../tool_test.go | 4 +- .../tool.go | 16 +- .../tool_test.go | 6 +- pkg/tools/edge_find_capabilities/tool.go | 6 +- pkg/tools/edge_find_capabilities/tool_test.go | 2 +- .../tool.go | 10 +- .../tool_test.go | 4 +- pkg/tools/edge_get_capability/tool.go | 6 +- pkg/tools/edge_get_capability/tool_test.go | 2 +- pkg/tools/edge_get_connection/tool.go | 4 +- pkg/tools/edge_get_job_status/tool.go | 49 +++++ pkg/tools/edge_get_job_status/tool_test.go | 46 +++++ pkg/tools/edge_get_job_status_history/tool.go | 4 +- pkg/tools/edge_list_capabilities/tool.go | 4 +- .../tool.go | 12 +- .../tool_test.go | 4 +- pkg/tools/edge_list_connections/tool.go | 4 +- .../tool.go | 12 +- .../tool_test.go | 4 +- pkg/tools/find_domain_types/tool.go | 2 +- pkg/tools/get_catalog_job_status/tool.go | 65 ------- pkg/tools/get_catalog_job_status/tool_test.go | 41 ----- pkg/tools/get_data_source_setup_guide/tool.go | 12 +- pkg/tools/get_job_status/tool.go | 36 +++- pkg/tools/get_job_status/tool_test.go | 21 +-- pkg/tools/jobs_find/tool.go | 6 +- pkg/tools/list_integrations/tool.go | 26 +-- pkg/tools/list_integrations/tool_test.go | 6 +- pkg/tools/register.go | 74 ++++---- pkg/tools/start_ingestion/tool.go | 8 +- pkg/tools/test_connection/tool.go | 10 +- pkg/tools/upload_file/tool.go | 10 +- 82 files changed, 1169 insertions(+), 389 deletions(-) create mode 100644 pkg/skills/files/collibra/etl-integration/SKILL.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/azure-foundry.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/azureml.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/bedrock.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/databricks-uc.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/dataplex.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/datazone.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/fabric.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/index.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/mlflow.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/purview.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/sagemaker.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/sigma.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/thoughtspot.md create mode 100644 pkg/skills/files/collibra/etl-integration/references/vertex.md rename pkg/tools/{catalog_generic_add_schedule => catalog_etl_add_schedule}/tool.go (89%) rename pkg/tools/{catalog_generic_add_schedule => catalog_etl_add_schedule}/tool_test.go (94%) rename pkg/tools/{catalog_generic_cancel_job => catalog_etl_cancel_job}/tool.go (90%) rename pkg/tools/{catalog_generic_cancel_job => catalog_etl_cancel_job}/tool_test.go (93%) rename pkg/tools/{catalog_generic_delete_config => catalog_etl_delete_generic_config}/tool.go (84%) rename pkg/tools/{catalog_generic_delete_config => catalog_etl_delete_generic_config}/tool_test.go (93%) rename pkg/tools/{catalog_generic_delete_schedule => catalog_etl_delete_schedule}/tool.go (86%) rename pkg/tools/{catalog_generic_delete_schedule => catalog_etl_delete_schedule}/tool_test.go (93%) rename pkg/tools/{catalog_generic_get_all_schedules => catalog_etl_get_all_schedules}/tool.go (86%) rename pkg/tools/{catalog_generic_get_all_schedules => catalog_etl_get_all_schedules}/tool_test.go (93%) rename pkg/tools/{catalog_generic_get_config => catalog_etl_get_config}/tool.go (81%) rename pkg/tools/{catalog_generic_get_config => catalog_etl_get_config}/tool_test.go (94%) rename pkg/tools/{catalog_generic_get_schedule => catalog_etl_get_schedule}/tool.go (82%) rename pkg/tools/{catalog_generic_get_schedule => catalog_etl_get_schedule}/tool_test.go (94%) rename pkg/tools/{catalog_generic_get_schema => catalog_etl_get_schema}/tool.go (82%) rename pkg/tools/{catalog_generic_get_schema => catalog_etl_get_schema}/tool_test.go (94%) rename pkg/tools/{catalog_generic_save_config => catalog_etl_save_generic_config}/tool.go (82%) rename pkg/tools/{catalog_generic_save_config => catalog_etl_save_generic_config}/tool_test.go (94%) rename pkg/tools/{catalog_generic_start_job => catalog_etl_start_job}/tool.go (89%) rename pkg/tools/{catalog_generic_start_job => catalog_etl_start_job}/tool_test.go (94%) rename pkg/tools/{catalog_generic_update_schedule => catalog_etl_update_schedule}/tool.go (88%) rename pkg/tools/{catalog_generic_update_schedule => catalog_etl_update_schedule}/tool_test.go (94%) rename pkg/tools/{create_capability => edge_create_capability}/tool.go (78%) rename pkg/tools/{create_capability => edge_create_capability}/tool_test.go (96%) rename pkg/tools/{create_connection => edge_create_connection}/tool.go (90%) rename pkg/tools/{create_connection => edge_create_connection}/tool_test.go (97%) rename pkg/tools/{find_connections => edge_find_connections}/tool.go (86%) rename pkg/tools/{find_connections => edge_find_connections}/tool_test.go (93%) create mode 100644 pkg/tools/edge_get_job_status/tool.go create mode 100644 pkg/tools/edge_get_job_status/tool_test.go rename pkg/tools/{list_capability_types => edge_list_capability_types}/tool.go (90%) rename pkg/tools/{list_capability_types => edge_list_capability_types}/tool_test.go (97%) rename pkg/tools/{list_edge_sites => edge_list_sites}/tool.go (73%) rename pkg/tools/{list_edge_sites => edge_list_sites}/tool_test.go (91%) delete mode 100644 pkg/tools/get_catalog_job_status/tool.go delete mode 100644 pkg/tools/get_catalog_job_status/tool_test.go diff --git a/.gitignore b/.gitignore index 50fff47..2136701 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build mcp.yaml oas/** mise.toml +.DS_Store diff --git a/README.md b/README.md index deaefe7..dcd6592 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,15 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`discover_business_glossary`](pkg/tools/discover_business_glossary/) - Ask questions about terms and definitions. Note that this tool leverages Collibra AI and therefore consumes Collibra Units (CUs). **Requires:** `dgc.ai-copilot` - [`discover_data_assets`](pkg/tools/discover_data_assets/) - Query available data assets using natural language. Note that this tool leverages Collibra AI and therefore consumes Collibra Units (CUs). **Requires:** `dgc.ai-copilot` -- [`find_connections`](pkg/tools/find_connections/) - Find existing Edge connections by name, to reuse instead of duplicating. **Requires:** `dgc.edge-view-connections-and-capabilities` +- [`edge_find_connections`](pkg/tools/edge_find_connections/) - Find existing Edge connections by name, to reuse instead of duplicating. **Requires:** `dgc.edge-view-connections-and-capabilities` - [`find_domain_types`](pkg/tools/find_domain_types/) - Resolve a domain type name (e.g. "Physical Data Dictionary") to its UUID - [`find_users`](pkg/tools/find_users/) - Resolve a user name to their UUID, e.g. for a database asset's owners - [`get_asset_details`](pkg/tools/get_asset_details/) - Retrieve detailed information about specific assets by UUID, including the asset's assignable attribute schema (every attribute it can hold, including empty ones) - [`get_business_term_data`](pkg/tools/get_business_term_data/) - Trace a business term back to its connected physical data assets -- [`get_catalog_job_status`](pkg/tools/get_catalog_job_status/) - Poll the status of a DGC catalog job, e.g. a jdbc-ingestion sync started by `start_ingestion` +- [`get_job_status`](pkg/tools/get_job_status/) - Poll the status of a DGC job, e.g. a jdbc-ingestion sync started by `start_ingestion` - [`get_column_semantics`](pkg/tools/get_column_semantics/) - Retrieve data attributes, measures, and business assets connected to a column - [`get_data_source_setup_guide`](pkg/tools/get_data_source_setup_guide/) - Look up JDBC driver class, connection string format, and connection properties for a data source from Collibra's documentation -- [`get_job_status`](pkg/tools/get_job_status/) - Poll the status of an Edge job, e.g. a connection test or capability run +- [`edge_get_job_status`](pkg/tools/edge_get_job_status/) - Poll the status of an Edge job, e.g. a connection test or capability run - [`get_lineage_downstream`](pkg/tools/get_lineage_downstream/) - Get downstream technical lineage (consumers) for a data entity - [`get_lineage_entity`](pkg/tools/get_lineage_entity/) - Get metadata about a specific entity in the technical lineage graph - [`get_lineage_transformation`](pkg/tools/get_lineage_transformation/) - Get details and logic of a specific data transformation @@ -28,9 +28,9 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`get_measure_data`](pkg/tools/get_measure_data/) - Trace a measure back to its underlying physical columns and tables - [`get_table_semantics`](pkg/tools/get_table_semantics/) - Retrieve the semantic layer for a table: columns, data attributes, and connected measures - [`list_asset_types`](pkg/tools/list_asset_types/) - List available asset types -- [`list_capability_types`](pkg/tools/list_capability_types/) - List capability and connection types available on an Edge site, with their manifests. **Requires:** `dgc.edge-view-connections-and-capabilities` +- [`edge_list_capability_types`](pkg/tools/edge_list_capability_types/) - List capability and connection types available on an Edge site, with their manifests. **Requires:** `dgc.edge-view-connections-and-capabilities` - [`list_data_contract`](pkg/tools/list_data_contracts/) - List data contracts with pagination -- [`list_edge_sites`](pkg/tools/list_edge_sites/) - List available Edge sites. **Requires:** `dgc.edge-view-connections-and-capabilities` +- [`edge_list_sites`](pkg/tools/edge_list_sites/) - List available Edge sites. **Requires:** `dgc.edge-view-connections-and-capabilities` - [`prepare_create_asset`](pkg/tools/prepare_create_asset/) - Read-only companion to `create_asset`: enumerate available asset types and domains, resolve a UUID/publicId/displayName for either, and hydrate the scoped attribute and relation schema for a chosen pair - [`pull_data_contract_manifest`](pkg/tools/pull_data_contract_manifest/) - Download manifest for a data contract - [`search_asset_keyword`](pkg/tools/search_asset_keyword/) - Wildcard keyword search for assets; filters (status, community, domain, domain type, asset type, created-by) accept names or UUIDs @@ -45,9 +45,9 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`add_data_classification_match`](pkg/tools/add_data_classification_match/) - Associate a data class with an asset. **Requires:** `dgc.classify`, `dgc.catalog` - [`configure_database`](pkg/tools/configure_database/) - Register a database and configure its schema synchronization rules for jdbc-ingestion - [`create_asset`](pkg/tools/create_asset/) - Create a new asset of any type. Resolves `assetType` (UUID, publicId, or display name), `domain` (UUID or name), `status` (UUID or name), and attributes (by name or typeId) server-side; converts Markdown to HTML for `RICH_TEXT` attributes; gates on duplicate-name (default `allowDuplicate: false`) -- [`create_capability`](pkg/tools/create_capability/) - Create or update an Edge capability (e.g. jdbc-ingestion). **Requires:** `dgc.edge-integration-capability-manage` +- [`edge_create_capability`](pkg/tools/edge_create_capability/) - Create or update an Edge capability (e.g. jdbc-ingestion). **Requires:** `dgc.edge-integration-capability-manage` - [`create_community`](pkg/tools/create_community/) - Create a DGC community -- [`create_connection`](pkg/tools/create_connection/) - Create or update an Edge connection (e.g. a JDBC connection). **Requires:** `dgc.edge-integration-capability-manage` +- [`edge_create_connection`](pkg/tools/edge_create_connection/) - Create or update an Edge connection (e.g. a JDBC connection). **Requires:** `dgc.edge-integration-capability-manage` - [`create_domain`](pkg/tools/create_domain/) - Create a DGC domain within a community - [`edit_asset`](pkg/tools/edit_asset/) - Edit an existing asset via a list of typed operations: - `set_attribute`, `add_attribute`, `remove_attribute` - set an attribute value (creates if empty, updates if present), append an extra value to a multi-valued attribute, or clear one (e.g. `Definition`, `Note`) diff --git a/SKILLS.md b/SKILLS.md index fa08444..2fbc277 100644 --- a/SKILLS.md +++ b/SKILLS.md @@ -18,6 +18,7 @@ those tools. Skill content lives in [`pkg/skills/files/collibra/`](pkg/skills/fi | `collibra/asset-create` | `create_asset` workflow; RICH_TEXT Markdown handling; duplicate gating | | `collibra/asset-edit` | `edit_asset` operation types | | `collibra/jdbc-ingestion` | End-to-end jdbc-ingestion setup: Edge connection/capability, database registration, running and polling a sync | +| `collibra/etl-integration` | End-to-end ETL sync integrations (Dataplex, Databricks UC, Purview, Sigma, …): connection, capability, generic config (fetched live), schedule/run; per-integration references | Each skill is one `SKILL.md` per directory, with frontmatter (`description`, `related`) and an optional `references/` directory for bundled reference documents. diff --git a/pkg/clients/data_source_docs_client.go b/pkg/clients/data_source_docs_client.go index d69e5d4..587e373 100644 --- a/pkg/clients/data_source_docs_client.go +++ b/pkg/clients/data_source_docs_client.go @@ -130,7 +130,7 @@ func GetDataSourceInfo(ctx context.Context, client *http.Client, slug string) (* // source, grouped by authentication method (e.g. a Snowflake connection documents // separate property sets for "Username and password", "Key pair (unencrypted)", // "OAuth", etc.). Use this to know exactly which named properties to pass as -// create_connection's additionalProperties for a given auth method. +// edge_create_connection's additionalProperties for a given auth method. func GetConnectionProperties(ctx context.Context, client *http.Client, slug string) ([]AuthMethodProperties, error) { doc, err := fetchAndParseHTML(ctx, client, catalogConnectorsBaseURL+"/"+slug+"/minimum-connection-properties-edge.htm") if err != nil { diff --git a/pkg/clients/dgc_job_client.go b/pkg/clients/dgc_job_client.go index f6dabdf..bbe7e11 100644 --- a/pkg/clients/dgc_job_client.go +++ b/pkg/clients/dgc_job_client.go @@ -12,7 +12,7 @@ import ( // This is the DGC-wide job resource you poll for status (e.g. after triggering a // catalog database sync, whose CatalogJob response carries the id to poll here). It is // distinct from edge-management's per-Edge-site jobs (see GetJobStatusLog / -// get_job_status), which track Edge capability runs and connection tests instead. +// edge_get_job_status), which track Edge capability runs and connection tests instead. type Job struct { ID string `json:"id"` Name string `json:"name,omitempty"` diff --git a/pkg/clients/generic_integration_client.go b/pkg/clients/generic_integration_client.go index 456eb55..96d501e 100644 --- a/pkg/clients/generic_integration_client.go +++ b/pkg/clients/generic_integration_client.go @@ -13,9 +13,9 @@ const genericIntegrationBasePath = "/rest/catalog/1.0/genericIntegration" // --- Types --- type GenericConfiguration struct { - Id string `json:"id,omitempty"` + Id string `json:"id,omitempty"` IngestibleId string `json:"ingestibleId,omitempty"` - Value string `json:"value,omitempty"` + Value string `json:"value,omitempty"` } type SaveGenericConfigRequest struct { @@ -23,13 +23,13 @@ type SaveGenericConfigRequest struct { } type GenericSchedule struct { - Id int64 `json:"id,omitempty"` - CronExpression string `json:"cronExpression,omitempty"` - CronTimeZone string `json:"cronTimeZone,omitempty"` - LastRunTimeStamp int64 `json:"lastRunTimeStamp,omitempty"` - NextRunDateLongValue int64 `json:"nextRunDateLongValue,omitempty"` - CronJson string `json:"cronJson,omitempty"` - Workflow string `json:"workflow,omitempty"` + Id int64 `json:"id,omitempty"` + CronExpression string `json:"cronExpression,omitempty"` + CronTimeZone string `json:"cronTimeZone,omitempty"` + LastRunTimeStamp int64 `json:"lastRunTimeStamp,omitempty"` + NextRunDateLongValue int64 `json:"nextRunDateLongValue,omitempty"` + CronJson string `json:"cronJson,omitempty"` + Workflow string `json:"workflow,omitempty"` } type AddGenericScheduleRequest struct { @@ -45,19 +45,19 @@ type ChangeGenericScheduleRequest struct { } type GenericJob struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - State string `json:"state,omitempty"` - Result string `json:"result,omitempty"` - Message string `json:"message,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + State string `json:"state,omitempty"` + Result string `json:"result,omitempty"` + Message string `json:"message,omitempty"` ProgressPercentage float64 `json:"progressPercentage,omitempty"` - Cancelable bool `json:"cancelable,omitempty"` - StartDate int64 `json:"startDate,omitempty"` - EndDate int64 `json:"endDate,omitempty"` - CreatedBy string `json:"createdBy,omitempty"` - CreatedOn int64 `json:"createdOn,omitempty"` - UserId string `json:"userId,omitempty"` + Cancelable bool `json:"cancelable,omitempty"` + StartDate int64 `json:"startDate,omitempty"` + EndDate int64 `json:"endDate,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + CreatedOn int64 `json:"createdOn,omitempty"` + UserId string `json:"userId,omitempty"` } type genericScheduleParams struct { @@ -276,4 +276,4 @@ func StartGenericJob(ctx context.Context, client *http.Client, ingestibleId stri return nil, fmt.Errorf("failed to parse job response: %w", err) } return &job, nil -} \ No newline at end of file +} diff --git a/pkg/clients/prepare_create_asset_client.go b/pkg/clients/prepare_create_asset_client.go index 0d9cf83..1ff9e15 100644 --- a/pkg/clients/prepare_create_asset_client.go +++ b/pkg/clients/prepare_create_asset_client.go @@ -61,17 +61,16 @@ type PrepareCreateDomainListResponse struct { Total int `json:"total"` } - // PrepareCreateAttributeType represents an attribute type with full schema. type PrepareCreateAttributeType struct { - ID string `json:"id"` - Name string `json:"name"` - Kind string `json:"kind"` - Required bool `json:"required"` - Constraints *PrepareCreateConstraints `json:"constraints,omitempty"` - AllowedValues []string `json:"allowedValues,omitempty"` - Direction string `json:"direction,omitempty"` - TargetAssetType *PrepareCreateAssetType `json:"targetAssetType,omitempty"` + ID string `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` + Required bool `json:"required"` + Constraints *PrepareCreateConstraints `json:"constraints,omitempty"` + AllowedValues []string `json:"allowedValues,omitempty"` + Direction string `json:"direction,omitempty"` + TargetAssetType *PrepareCreateAssetType `json:"targetAssetType,omitempty"` } // PrepareCreateConstraints represents attribute validation constraints. diff --git a/pkg/skills/files/collibra/etl-integration/SKILL.md b/pkg/skills/files/collibra/etl-integration/SKILL.md new file mode 100644 index 0000000..231ab90 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/SKILL.md @@ -0,0 +1,172 @@ +--- +description: Set up, configure, schedule, and run Edge ETL integrations (metadata sync capabilities like Dataplex, Databricks Unity Catalog, Purview, Sigma) end to end. +related: collibra/jdbc-ingestion, collibra/discovery +--- + +# ETL integrations — connect, configure, schedule, run + +An ETL integration is an Edge **capability**: a small app running in the Edge Kubernetes +cluster that extracts metadata from a third-party system and loads it into the Collibra +Data Governance Platform (DGC). Setup spans two systems — **Edge** (connection + +capability) and **DGC catalog** (the generic config + schedule/run). No single tool does +the whole thing; calling them out of order is the most common way this fails. + +## First: confirm this is the right ingestion path + +Many data sources support **more than one** ingestion path — most commonly a native +**ETL integration** (this skill) *and* generic **JDBC** (`collibra/jdbc-ingestion`). Which +one to use is a real decision, not a default. + +**Whenever the user asks to create an integration, before doing anything else, determine +which ingestion type it should be** — don't assume ETL just because this skill is loaded: + +1. Identify the data source. +2. Check whether more than one path supports it (`references/index.md` lists the data + sources with a native ETL integration; anything reachable over a JDBC driver can also go + through `collibra/jdbc-ingestion`). +3. **If more than one path is possible, ask the user which they want** before starting. + Only proceed with this ETL skill once the user has chosen it (or it's the only supported + path). + +## The three configuration layers (merged at run time) + +When a run starts, DGC's catalog module merges three parameter sets into one and sends +them to the Edge site to start the capability's workflow; it also creates a **Job** in +DGC to track progress and, on completion, records status metrics and a report. + +1. **Edge connection** — credentials/endpoint for the third-party system. + `edge_create_connection`. +2. **Edge capability** — the integration instance and its capability parameters (most are + shared across capabilities, some are specific). `edge_create_capability`. **Its id is + the `ingestibleId`** you use for every catalog step below. +3. **Generic (ETL) config** — the sync configuration specific to the integration. + `catalog_etl_save_generic_config`. + +## The full sequence + +1. **`edge_list_sites`** → the `edgeSiteId` you'll reference below. +2. **`edge_list_capability_types`** (pass a `query`) → find the connection type and + capability type for your integration and read their parameter manifests. Don't guess + parameter names — read them here. +3. **`edge_create_connection`** → the credentials for the third-party system. +4. **`edge_create_capability`** → `typeId` = the capability type from step 2; + `parameters.connection` = the connection id from step 3. **The returned capability id + is your `ingestibleId`.** +5. **`catalog_etl_get_schema(ingestibleId)`** → the JSON schema for the generic config. + **Always fetch this and shape the config to match it — never hardcode or guess field + names.** (See "Never hardcode the config schema".) +5a. **Resolve DGC prerequisites** — nearly every integration's generic config requires at + least a `domain` UUID; many also require a `community` or a `system` (System asset) UUID. + **Before building the config, ask the user** whether they want to use an existing + domain/System or create new ones. (See "DGC prerequisites".) +6. **`catalog_etl_save_generic_config(ingestibleId, configuration)`** → the config JSON, + conforming to the schema from step 5. Read it back any time with + **`catalog_etl_get_config`**. +7. Run or schedule: + - **`catalog_etl_start_job(ingestibleId[, workflow])`** — run immediately. + - **`catalog_etl_add_schedule(ingestibleId, cronExpression, cronTimeZone[, workflow])`** + — schedule for later. Inspect/update/remove with + `catalog_etl_get_schedule` / `catalog_etl_get_all_schedules` / + `catalog_etl_update_schedule` / `catalog_etl_delete_schedule`. +8. **Track:** poll the DGC job with **`get_job_status`** (find recent runs with + `jobs_find`); **`list_integrations`** gives the schedule + last/next run overview and is + the entry point for "show me all integrations" questions. Stop a running sync with + `catalog_etl_cancel_job`. + +## Workflows + +A capability has one or more workflows. **Inbound is the default** — omit the `workflow` +argument to target it. Some integrations expose multiple inbound workflows (e.g. different +entry types) and some an outbound workflow (DGC → third party). `catalog_etl_start_job`, +`catalog_etl_add_schedule`, `catalog_etl_update_schedule`, `catalog_etl_delete_schedule`, +and `catalog_etl_cancel_job` all take an optional `workflow` — omit it for the default +inbound, or set it to target a specific one. The **generic config is per-integration** +(one config via `get_schema`/`save_generic_config`/`get_config`), not per-workflow. + +## DGC prerequisites: domain, community, and System asset + +Most integrations require one or more of these DGC objects to already exist before +`catalog_etl_save_generic_config` can succeed. The exact required fields are in the schema +from `catalog_etl_get_schema` — but the pattern is consistent across integrations: + +| Field in generic config | DGC object | Notes | +|---|---|---| +| `domain` | Domain | Where ingested assets land. Almost universal. | +| `community` | Community | Required by some integrations (e.g. Dataplex CloudSQL, Purview). | +| `system` | System asset | Target System asset for the ingested data. Required by several integrations. | + +**Always ask the user before building the config:** + +> "Do you already have a domain/System you want to use, or should I create new ones?" + +### Using existing objects + +- Resolve a domain or System UUID with **`search_asset_keyword`** (filter by asset type + `"Domain"` or `"System"`) or **`prepare_create_asset`** (lists available domains and their + communities). + +### Creating new objects + +Follow this order — each step is a prerequisite for the next: + +1. **`create_community`** — if no suitable community exists yet. +2. **`create_domain`** — within the community. Use **`find_domain_types`** to resolve the + domain type name (e.g. `"Technology Asset Domain"`) to a `typeId` UUID first. +3. **`create_asset`** — to create a System asset, if required. Use **`prepare_create_asset`** + to resolve the `"System"` asset type UUID and pick the target domain before calling. + +## Never hardcode the config schema + +Each integration's generic-config schema lives in that integration's own repo (as a +react-json-schema-form data-schema) and evolves with the integration's version. This skill +deliberately **does not embed those schemas** — always call `catalog_etl_get_schema` at run +time. That endpoint returns the very schema DGC uses to validate +`catalog_etl_save_generic_config`, so it can never be stale or wrong for the customer's +version. Likewise, read connection/capability parameter names from +`edge_list_capability_types`, not from memory. The per-integration references capture the +stable conceptual detail (purpose, auth options, flows, gotchas) — not exact field lists. + +## Per-integration references + +For a specific integration, load its reference for auth types, flows, the capability/ +connection `typeId`s, and notable config nuances: `references/.md` (e.g. +`references/dataplex.md`). `references/index.md` lists them all. These **complement, not +replace,** the live schema from `catalog_etl_get_schema`. + +## Distinct from jdbc-ingestion + +`collibra/jdbc-ingestion` is a specialized flow: it registers a Database asset via +`configure_database` and runs via `start_ingestion` (DGC's `synchronizeMetadata`), **not** +`catalog_etl_start_job`. Use that skill for JDBC database ingestion; use **this** skill for +the generic/ETL sync capabilities (Dataplex, Databricks Unity Catalog, Purview, Sigma, +ThoughtSpot, …). + +## Not controllable via chip (yet) + +The cloud **object-storage** crawlers are **not** ETL integrations in the sense above and +**cannot be configured through chip** — they use different endpoints and a different +configuration model than the `catalog_etl_*` / edge tools here: + +- **AWS S3** (`s3-synchronization`) +- **Azure Data Lake Storage / ADLS** (`adls-synchronization`) +- **Google Cloud Storage / GCS** (`gcs-synchronization`) + +If a user asks to set up, configure, or run one of these, tell them it isn't supported via +chip yet and must be configured in the Collibra/Edge UI — do not attempt it with the tools +in this skill. + +## Hard rules + +1. **Fetch the config schema (`catalog_etl_get_schema`) before building the generic + config** — never guess field names or reuse a stale schema. +2. **Read connection/capability parameters from `edge_list_capability_types`,** not memory. +3. **Confirm scope with the user** (which projects/datasets/filters) — don't default to + "ingest everything." +4. **Two job-id spaces:** Edge jobs every started edge capability produce one → `edge_get_job_status`; + DGC jobs (from `catalog_etl_start_job`) → `get_job_status`. Never cross them. ETL capabilities utilize DGC jobs. Don't check Edge jobs unless directly asked +5. **Inbound only** unless the user explicitly asks to configure an outbound flow. +6. **Before building the generic config, ask the user** whether to use existing DGC + domain/community/System assets or create new ones — never assume they exist. See + "DGC prerequisites". +7. To fill in parameters ask user questions and before saving them confirm with the user. +8. Before creating capability confirm capability parameters with the user \ No newline at end of file diff --git a/pkg/skills/files/collibra/etl-integration/references/azure-foundry.md b/pkg/skills/files/collibra/etl-integration/references/azure-foundry.md new file mode 100644 index 0000000..b76f8da --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/azure-foundry.md @@ -0,0 +1,67 @@ +# Azure AI Foundry (`azure-foundry-synchronization`) + +Extracts AI asset metadata from Microsoft Azure AI Foundry (Azure Machine Learning +Workspaces, Azure Cognitive Services, Azure OpenAI) and ingests it into DGC — covering +AI projects, model deployments, AI agents, fine-tuned models, training files, AI +endpoints, and AI monitors. `capability-type` (the `typeId` for +`edge_create_capability`): **`azure-foundry-synchronization`**. + +**Docs:** [Steps overview: Integrate Azure AI Foundry via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/AzureAIFoundryModel/co_edge-integrate-azure-ai-foundry-steps.htm) + +**Connection type:** Azure. Auth methods (set on `edge_create_connection`): +- **Azure Service Principal (Client Credentials Flow)** — `clientId`, `clientSecret`, + `tenantId`. This is the only supported authentication method. + +The Service Principal needs `Reader` on the subscription plus `AzureML Data Scientist` +(or `Reader`) on each ML Workspace and `Cognitive Services Contributor` (or `Reader`) on +each Cognitive Services account. + +**Workflows:** inbound only, `main` (default — omit `workflow`). There is no outbound +(DGC → Azure) flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — UUID of the existing DGC domain where all assets will be created. + All workspaces share one domain; per-workspace domain routing is not supported. +- `workspaceNames` (required) — array of workspace identifiers in + `"resourceGroup/workspaceName"` format for ML workspaces, or + `"resourceGroup/accountName/projectName"` for Azure AI Foundry projects. At least one + entry is required. +- `associateAIUseCasesWithAIModels` (default `true`) — when true, creates `implementedBy` + relations between existing Collibra AI Use Case assets and ingested AI Model Version / + Base Model assets. +- `createFileAssets` (default `true`) — when true, ingests file metadata linked to AI + agents (training files, vector store files) and fine-tuning jobs as `File` and + `FileContainer` assets. +- `customAiMetricsMappings` (optional) — maps Azure Monitor metric names (e.g. + `TokenTransaction`, `RequestsPerMinute`) to Collibra attribute type UUIDs on + `AIModelDeployment` assets. Each entry: `{"from": "MetricName", "to": ""}`. The + UUID must exist as an attribute type in DGC or the job fails at validation before any + Azure data is fetched. +- `assetDeletionMode` (Edge capability parameter, default `Soft`) — `Soft` archives + missing assets; `Hard` permanently deletes them. +- `subscriptionId` (Edge capability parameter, required) — Azure Subscription ID + containing the AI Foundry workspaces. + +## Gotchas +- **Hub workspaces are expanded automatically.** If a workspace listed in `workspaceNames` + has `kind` containing `"hub"`, all child project workspaces are discovered and ingested + automatically — you do not need to list child projects individually. +- **All assets land in one domain.** There is no per-workspace or per-resource-group + domain routing. Choose the target domain carefully before the first run. +- **Each run is a full sync.** There is no incremental/delta mode. Every run fetches all + listed workspaces from scratch. +- **DGC version gates features.** Several asset types only appear on newer DGC versions: + `AIBaseModel`, `AIEndpoint`, `AIMonitor`, and `Vendor` assets require DGC 2026-03+ + (or 2026-04+ for Vendor). Fine-tuned model assets require DGC 2025-09+. The capability + enforces these limits at startup and skips unsupported assets rather than failing. +- **Fine-tuned model deployments auto-delete in Azure.** Azure removes inactive fine-tuned + model deployments after 15 days. The next sync will archive (Soft) or delete (Hard) the + corresponding `AIModelDeployment` asset in DGC. Schedule runs at least every 15 days to + keep DGC in sync. +- **`customAiMetricsMappings` UUIDs are validated before any Azure fetch.** If any `to` + UUID is not a valid attribute type in DGC, the job fails immediately with + `InvalidParametersException` before contacting Azure. +- **Metric attributes are not populated if Azure Monitor has no data.** If a deployment + has never received traffic, mapped metric attributes will remain unset. +- **Workspace names must match exactly.** An unrecognised workspace name in `workspaceNames` + throws `InvalidParametersException` before any data is fetched. diff --git a/pkg/skills/files/collibra/etl-integration/references/azureml.md b/pkg/skills/files/collibra/etl-integration/references/azureml.md new file mode 100644 index 0000000..50c9da2 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/azureml.md @@ -0,0 +1,24 @@ +# Azure Machine Learning (`azureml-synchronization`) + +Extracts metadata from Microsoft Azure Machine Learning workspaces and ingests it into Collibra DGC as **Azure AI Model Version** (`AzureAIModel`) assets, including performance metrics fetched from the embedded MLflow tracking server. + +**Docs:** [About the Azure ML integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/AzureAIModel/co_about-azure-ai.htm) + +**Connection type:** Azure (generic). Auth method (set on `edge_create_connection`): +- **Service Principal** — OAuth 2.0 Client Credentials flow; requires `clientId`, `clientSecret`, and `tenantId` for the Azure AD application. + +**Workflows:** single inbound workflow `main` (omit `workflow` argument — there is no outbound flow). + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — target Collibra Technology Asset Domain where `AzureAIModel` assets are created. +- `resourceGroupName` (required) — Azure resource group containing the workspace. +- `workspaceName` (required) — Azure ML workspace to synchronize; must be unique within the resource group. +- `customAiMetricsMappings` (optional) — maps MLflow metric keys (e.g., `f1_score_macro`) to Collibra attribute type IDs on the `AzureAIModel` asset type. The only metric auto-populated without a mapping is `accuracy` → `ModelAccuracy`; all other standard attributes (`ModelPrecision`, `MeanSquaredError`, `MeanAbsoluteError`) require an explicit mapping entry here. + +## Gotchas +- **Latest version only**: one `AzureAIModel` asset is created per model container, representing only the latest registered version. +- **AutoML models silently skipped**: models whose names match the AutoML child-run pattern are excluded without warning; if a workspace appears empty, verify it contains user-registered (non-AutoML) models. +- **Custom metric validation is eager**: if a `customAiMetricsMappings` entry references an attribute type not already assigned to the `AzureAIModel` asset type in Collibra, the synchronization aborts before fetching any data. Pre-assign the attribute types first. +- **No metrics when no job run**: models that were registered without an associated MLflow job run ID produce assets with name, version, and description only — no metric attributes are populated. +- **`InitiatingUserInSource` requires DGP 2026.07+**: on earlier platform versions this attribute is silently omitted even when present in the source. +- **Azure permissions**: the service principal needs `Microsoft.MachineLearningServices/workspaces/models/versions/read` and `workspaces/jobs/read`. The built-in **AzureML Data Scientist** role covers both; a plain **Reader** role retrieves models but no MLflow metrics. diff --git a/pkg/skills/files/collibra/etl-integration/references/bedrock.md b/pkg/skills/files/collibra/etl-integration/references/bedrock.md new file mode 100644 index 0000000..61a57c5 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/bedrock.md @@ -0,0 +1,42 @@ +# AWS Bedrock AI (`bedrock-synchronization`) + +Extracts AI metadata from AWS Bedrock (foundation models, custom/imported model +versions, deployments, endpoints, agents, vendors, and — when data ingestion is +enabled — the S3 training datasets and invocation-log output locations) and ingests +it into DGC. `capability-type` (the `typeId` for `edge_create_capability`): +**`bedrock-synchronization`**. + +**Docs:** [About the AWS Bedrock AI integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/AWSBedrock/co_about-aws-bedrock-ai.htm) + +**Connection type:** AWS (`family: generic`). Auth methods (set on `edge_create_connection` +via the `authentication` property): +- **IAM static credentials** — `authentication = IAM` plus `aws_access_key_id` / + `aws_secret_access_key`; for Edge running outside AWS. +- **Instance / Pod role** — any other `authentication` value; keys left blank, resolves + the EC2 instance role or EKS IRSA role via the AWS default credentials chain. + +**Workflows:** inbound only, `main` (default — omit `workflow`). There is no outbound +(DGC → Bedrock) flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — target domain UUID; all catalogued assets are placed here. +- `regions` — AWS regions to scan; if empty, every region where Bedrock is generally + available is scanned (only regions actually offering Bedrock return results). +- `ingestAiInputAndOutput` (default true) — also ingest S3 File/File Container assets for + custom-model training data and deployment invocation-log output; when false, only model + metadata is ingested (cheaper, fewer AWS calls). +- `customAiMetricsMappings` — map a custom Bedrock training/validation metric name to a + Collibra AI Model attribute type; without mappings no metric values are persisted. + +## Gotchas +- IAM mode requires **both** keys — the run fails with `InvalidParametersException` if + either is missing while `authentication = IAM`. +- Per-region resilience: a region returning 4xx is marked "without access" and a region + where Bedrock isn't deployed (`UnknownHostException`) is marked "without Bedrock"; the + run continues and reports **Completed with Errors** if at least one region returned data, + or **Failed** only if every region was rejected. +- `ingestAiInputAndOutput` triggers extra AWS calls (`GetCustomModel`, + `GetModelInvocationLoggingConfiguration`) — CloudWatch-only logging is skipped, and + missing per-model permissions degrade gracefully (model imported without metrics). +- `save-input-metadata` uploads a ZIP of raw AWS responses to DGC; enable only at Collibra + Support's request, as it snapshots all metadata visible to the AWS principal. diff --git a/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md b/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md new file mode 100644 index 0000000..12a9ff7 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md @@ -0,0 +1,32 @@ +# Technical Lineage for Databricks Unity Catalog (`databricks-lineage`) + +Extracts data lineage from the Databricks Unity Catalog lineage system tables and produces column-level technical lineage in DGC, stitched to existing Databricks assets. +`capability-type` (the `typeId` for `edge_create_capability`): **`databricks-lineage`**. + +**Docs:** [Add the technical lineage for Databricks Unity Catalog capability](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/DatabricksUnityCatalog/ta_databricks-add-capability.htm) + +**Connection type:** Databricks (requires `workspaceUrl`). Auth methods (set via `authType` on `edge_create_connection`): +- **Personal Access Token** — `accessToken`. +- **OAuth** — `clientId` + `clientSecret`. +- **Microsoft Entra ID** — `clientId` + `clientSecret` + `tenantId`. + +**Workflows:** inbound is `main` (default — omit `workflow`). No outbound flow; instead, running with the `active` capability parameter set to `false` for a `sourceId` removes that source's lineage from the Techlin server. + +## Capability parameters (set on `edge_create_capability`) +- `httpPath` (required) — HTTP path of the SQL warehouse / compute resource (must start with `sql`). +- `sourceId` (required) — unique lineage source id; also the key for the exclusion flow. +- `processingLevel` — `Analyze` (load + analyze) or `Sync` (also synchronize to DGC). +- `timeFrame` — lookback window in days (default 365). + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `system` (required) — target System asset the ingested objects are stitched to. +- `includeFilters` / `excludeFilters` — scope by `Workspace > Catalog > Schema` (wildcards `*`/`?`; exclude wins). +- `catalog` — read lineage from views in a custom catalog instead of the `system` catalog. +- `sqlSourcesLimit` / `ingestSqlSources` — include SQL transformation texts, capped per relation. +- `ingestExternalLocations`, `ingestVolumes`, `ingestEntities` — extend coverage to external locations, volumes, and notebook/entity transformations. + +## Gotchas +- This capability does **not** create the Databricks assets — they must already exist in DGC (ingested by the separate `databricks-edge-capability`) for stitching to work; stitching matches on the exact full path `(System >) Database > Schema > Table > Column`. +- Set `useCollibrasystemName` when the same paths exist across environments (e.g. PROD/DEV), otherwise stitching is non-deterministic; changing it requires re-analyzing all sources. +- Requires read permissions on the Unity Catalog lineage system tables (`system.access.column_lineage`, `system.access.table_lineage`, `system.query.history`) plus compute attach rights. +- Lineage retention is fixed at 365 days by Databricks; large datasets rely on the default `scalable` query strategy (tune via `jvm-args`). diff --git a/pkg/skills/files/collibra/etl-integration/references/databricks-uc.md b/pkg/skills/files/collibra/etl-integration/references/databricks-uc.md new file mode 100644 index 0000000..285d8d3 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/databricks-uc.md @@ -0,0 +1,47 @@ +# Databricks Unity Catalog (`databricks-edge-capability`) + +Extracts metadata from Databricks Unity Catalog (catalogs, schemas, tables, columns, volumes, AI models/deployments/endpoints) and ingests it into DGC. The `typeId` for `edge_create_capability` is **`databricks-edge-capability`**. + +**Docs:** [About the Databricks Unity Catalog integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/Databricks/co_about-databricks-integration.htm) + +**Connection type:** Databricks. Auth methods (set on `edge_create_connection`): +- **Personal Access Token** — `accessToken` field. +- **OAuth** — `clientId` + `clientSecret`. +- **Microsoft Entra ID** — `clientId` + `clientSecret` + `tenantId`. + +**Workflows:** +- `main` (default) — combined inbound; flow type is chosen by `ingestionType` in the generic config. +- `ai` — dedicated AI asset inbound (models, deployments, endpoints, agents, monitors; DGC 2026.03+). +- `ai-monitoring` — reads MLflow trace metrics via JDBC and POSTs them to the Collibra AI Governance API; does **not** create or modify DGC assets. +- `outbound` — writes DGC attribute changes back to Databricks Unity Catalog as tags. + +## Ingestion modes (selected in the generic config, NOT via the workflow arg) + +Applies when using the `main` (default) workflow via the `ingestionType` config field: + +- **Metadata ingestion** — standard catalog/schema/table/column structure, JDBC-compatible with other ingestion capabilities. Requires `system`. +- **AI model ingestion** (Deprecated) — ingests AI base models, versions, deployments, endpoints, agents, monitors. Requires `domain`. +- **Metadata and AI model ingestion** (Deprecated) — combined; requires both `system` and `domain`. + +When using the dedicated `ai` workflow, `ingestionType` is not used — the workflow name itself determines the flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) + +- `system` — required for metadata flows; the System asset in DGC that becomes the root for ingested catalog/schema/table assets. +- `domain` — required for AI model flows; the DGC domain where AI assets land. +- `domainIncludeMappings` / `domainExcludeMappings` — scope ingestion to specific catalogs, schemas, or tables using `catalog > schema > table` patterns (wildcards `*` and `?` supported). Empty = ingest everything the connection can see. +- `version` — domain mapping resolution version. `V0` (default): unmatched schemas inherit the catalog domain. `V1`: unmatched schemas are excluded. New configurations should use `V1`. +- `ingestVolumes` — when enabled, also ingests Databricks Volume assets (DatabricksVolume, Directory, File) for each schema. Requires the Databricks Files API to be enabled on the workspace. +- `tagAttributeMappings` — routes Unity Catalog tag values to specific DGC attributes in addition to (or instead of) the default `Source Tags` attribute. +- `extensiblePropertiesMapping` — maps Databricks system properties (e.g. `table_type`, `created_at`, `metastore_id`) and custom table parameters to DGC attributes. +- `ingestInputDatasetsAndDeploymentOutput` (AI flow) — enables ingestion of MLflow training datasets and inference table output linked to model versions/deployments. + +## Gotchas + +- **JDBC connection is required for tag fetching.** A separate `jdbc-connection` capability parameter must reference a JDBC-type Edge connection. Without it, Unity Catalog tags are not fetched and `Source Tags` will be empty. The same JDBC connection also enables Profiling and Classification. +- **Domain cannot be changed for existing assets.** Once a catalog (Database) or schema asset has been ingested, its domain is locked — the engine looks up existing assets by name and reuses their domain regardless of the current mapping. Get domain mappings right before the first run. +- **Do not configure AI ingestion in both tabs.** Providing AI parameters in both the "Metadata Inbound" tab and the dedicated "AI ingestion" tab causes synchronization to fail on DGC < 2026.04. +- **Governed tags in Databricks only accept predefined values.** The outbound flow silently skips tag writes whose value is not permitted by the tag policy; the DGC attribute is updated but the Databricks tag stays unchanged. Check outbound logs for errors when mapping governed tags. +- **Volumes ingestion requires a Databricks beta flag.** The Files API (`enable_experimental_files_api_client = True`) must be enabled on the workspace before `ingestVolumes` will work. +- **`ai-monitoring` produces no DGC catalog assets.** It writes only to the AI Governance API (`/rest/aiGovernance/v1/metrics/bulk`). There is no synchronization report entry; success/failure is reported as an Edge job status only. +- **`stopComputeResource` shuts down the compute cluster after tag extraction.** Only relevant when `httpPath` is set. Enable with caution in shared-cluster environments. diff --git a/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md b/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md new file mode 100644 index 0000000..f21fb19 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md @@ -0,0 +1,32 @@ +# Technical Lineage for GCP (`dataplex-lineage-synchronization`) + +Extracts technical lineage from Google Dataplex (Data Lineage API) plus BigQuery / GCS and pushes it to Collibra Data Lineage (Techlin), producing table- and column-level technical lineage in DGP. This is a lineage capability — it does **not** ingest table/column assets; those must already exist in DGP for stitching to work. +`capability-type` (the `typeId` for `edge_create_capability`): **`dataplex-lineage-synchronization`**. + +**Docs:** [Synchronize technical lineage for Google Dataplex](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/GoogleDataplex/ta_dataplex-sync-techlin.htm) + +**Connection type:** GCP. Auth methods (set on `edge_create_connection`): +- **Service Account** — full JSON key (`gcp_service_account_credentials_json`). +- **Workload Identity Federation (WIF)** — federated identity; JSON only for file-based WIF. +- **WIF using GKE** — when the Edge site runs in GKE; uses the site's federated identity. + +**Workflows:** inbound only (default — omit `workflow`). No outbound flow. Setting the capability param `active=false` for a `sourceId` triggers removal of that source's lineage from Techlin instead of generating any. + +## Ingestion modes (selected in the generic config, NOT via the workflow arg) +- **Table lineage** (`lineageType` default) — table-level lineage; supports `includeFilters` / `excludeFilters`. +- **Column lineage** — column-level lineage; requires `gcsBucket` and the (preview) Export Metadata API. Note: stitching is not supported at table level, only column level. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `system` (required) — target System asset the ingested objects are stitched to. +- `projects` — GCP project IDs where Dataplex lineage is enabled (required for WIF; else derived from the SA key). +- `locations` — GCP regions to harvest from; defaults to the supported Dataplex regions. +- `lineageType` — Table vs Column lineage (see modes above). +- `gcsBucket` — required for Column lineage; path to the GCS bucket holding exported lineage. +- `includeFilters` / `excludeFilters` — Table-lineage only; patterns `project > location > @bigquery > dataset > table` with `*`/`?` wildcards; exclude wins over include. +- `sqlExtractionMethod` — `api` (default, one Get Job call per process) vs `query` (batched `INFORMATION_SCHEMA.JOBS_BY_PROJECT`). + +## Gotchas +- Stitching matches on the **exact full path** `(System >) Database > Schema > Table > Column`; assets must be pre-ingested (e.g. via BigQuery JDBC) or nothing stitches. GCS-file stitching is not supported. +- Enable `useCollibrasystemName` (capability param) when identical paths exist across environments (PROD/DEV) — otherwise stitching is non-deterministic. Changing it requires re-analyzing all sources. +- Dataplex retains lineage for only 30 days; older lineage is permanently gone, so runs must be scheduled recurringly. +- Column lineage depends on the preview `exportMetadata` API, which must be enabled by Google on request. diff --git a/pkg/skills/files/collibra/etl-integration/references/dataplex.md b/pkg/skills/files/collibra/etl-integration/references/dataplex.md new file mode 100644 index 0000000..52cdad4 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/dataplex.md @@ -0,0 +1,36 @@ +# Dataplex / Google Knowledge Catalog (`dataplex-synchronization`) + +Extracts metadata from Google Dataplex / Knowledge Catalog (formerly Dataplex Universal +Catalog) and ingests it into DGC. `capability-type` (the `typeId` for +`edge_create_capability`): **`dataplex-synchronization`**. + +**Docs:** [About Google Dataplex ingestion via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/GCS/Dataplex/co_about-dataplex.htm) + +**Connection type:** GCP. Auth methods (set on `edge_create_connection`): +- **Service Account** — full JSON key (`gcp_service_account_credentials_json`). +- **Workload Identity Federation (WIF)** — federated identity; JSON only for file-based WIF. +- **WIF using GKE** — when the Edge site runs in GKE; uses the site's federated identity. + +**Workflows:** inbound is `main` (default — omit `workflow`). Also has an outbound flow +(DGC → Dataplex). + +## Ingestion modes (selected in the generic config, NOT via the workflow arg) +- **DataSourceType**: `BigQuery` (default) or `CloudSQL` (MySQL / PostgreSQL / SQL Server). +- For BigQuery, **ingestionType** picks the ETL flow: **Knowledge Catalog** ingestion + (JDBC-compatible db/schema/table/column structure) or legacy **Dataplex** ingestion + (older non-JDBC APIs, still supported). + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `system` (required) — target System asset; `community` required for CloudSQL. +- `projects` / `locations` (optional) — scope ingestion; empty = all the connection can see. +- `domainIncludeMappings` / `domainExcludeMappings` + `domainMappingVersion` (V0 keeps an + existing asset's domain; V1 lets the mapping override it). +- `aspectMappings` (Knowledge Catalog) / `customLabelMappings` (legacy Dataplex) — map + source aspects/labels to DGC attributes. +- `columnsIngestionMode` — parent-only / parent+nested / flattened column structures. + +## Gotchas +- The BigQuery-vs-CloudSQL and Knowledge-Catalog-vs-legacy choice lives in the **config** + (`DataSourceType`/`ingestionType`), not the Edge `workflow` argument. +- An optional `jdbc-connection` capability parameter only enables profiling/classification + of ingested data — it is not the ingestion path itself. diff --git a/pkg/skills/files/collibra/etl-integration/references/datazone.md b/pkg/skills/files/collibra/etl-integration/references/datazone.md new file mode 100644 index 0000000..7a7d7fd --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/datazone.md @@ -0,0 +1,27 @@ +# AWS DataZone / SageMaker Unified Studio (`datazone-synchronization`) + +Extracts Redshift or Glue table/view metadata published in AWS SageMaker Unified Studio and ingests it into DGC as a System → Database → Schema → Table/View → Column hierarchy. The `typeId` for `edge_create_capability` is **`datazone-synchronization`**. + +**Docs:** [About the Amazon SageMaker Unified Studio integration](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/AwsSagemakeUnifiedStudioDataZone/co_about-aws-sagemaker-unified-studio.htm) + +**Connection type:** AWS (generic family). Auth methods (set on `edge_create_connection`): +- **IAM** — programmatic user credentials (`aws_access_key_id` + `aws_secret_access_key`). This is the only supported method. + +**Workflows:** inbound is `main` (default — omit `workflow`). Also has an outbound flow (DGC → SageMaker Unified Studio). + +## Ingestion modes (selected in the generic config, NOT via the workflow arg) +- **datasource**: `Amazon Redshift` (default) or `Amazon Glue`. Controls which asset type is pulled from the SageMaker Search API and determines the `DataSourceType` attribute on ingested Database assets. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `systemDomain` (required) — domain where System assets are created. +- `community` (required) — community under which per-database and per-schema sub-domains are auto-created. +- `datasource` (required) — `Amazon Redshift` or `Amazon Glue`. +- `regions` (required, array) — one or more AWS regions to scan; at least one must be provided. +- `datazoneDomains` (optional, array) — SageMaker Unified Studio domain IDs to include; empty means all domains visible to the connection. +- `jdbcConnectionsMapping` (optional, Redshift only) — array of `{ databaseName, connectionId }` pairs that link a Collibra Database full name to a JDBC connection, enabling profiling, classification, and sampling on that database. + +## Gotchas +- The IAM user must have `datazone:ListDomains`, `datazone:ListProjects`, and `datazone:SearchListings` permissions in every region you scan. +- `jdbcConnectionsMapping` is only relevant for Redshift. Glue tables are S3-backed; JDBC profiling is not supported in v1, so the field is hidden in the UI when `Amazon Glue` is selected. +- For Glue, all Glue databases within a catalog are mapped as schemas under a single synthetic `AwsDataCatalog` Database — there is no real per-catalog database concept in DataZone listings. +- The capability-level parameter `jdbc-capabilities-auto-create-mode` (set on `edge_create_capability`, not in the generic config) controls whether JDBC ingestion, profiling, and classification capabilities are created automatically when the integration runs. Defaults to auto-creating ingestion + profiling + classification. diff --git a/pkg/skills/files/collibra/etl-integration/references/fabric.md b/pkg/skills/files/collibra/etl-integration/references/fabric.md new file mode 100644 index 0000000..c8d14ec --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/fabric.md @@ -0,0 +1,57 @@ +# Microsoft Fabric (`fabric-synchronization`) + +Extracts metadata from Microsoft Fabric — traversing Tenant → Capacity → Workspace → +Lakehouses (schemas, tables, columns, files), Warehouses (schemas, tables, views, columns), +SQL Databases, and Mirrored Azure Databricks Catalogs — and ingests it into DGC. +`capability-type` (the `typeId` for `edge_create_capability`): **`fabric-synchronization`**. + +**Docs:** [About the Microsoft Fabric integration](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/MicrosoftFabric/co_about-fabric.htm) + +**Connection type:** Azure. Auth method (set on `edge_create_connection`): +- **Service Principal** — `client-id`, `client-secret`, and `tenant-id` of an Azure Entra + App registration (OAuth 2.0 client credentials flow). + +**Workflows:** inbound is `main` (default — omit `workflow`). No outbound flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `community` (required) — Collibra community where per-workspace Technology Assets + subdomains are auto-created. +- `mainDomain` (required) — domain where `MicrosoftTenant` and `FabricCapacity` assets land. +- `workspaceNames` (optional) — limit ingestion to specific workspace names; empty = all + accessible to the service principal. +- `maxLakehouseFiles` (optional, default `100`) — cap on files ingested per Lakehouse; set + to `-1` to ingest all files. +- `jdbcConnectionsMapping` (optional) — pairs `databaseName` (full Collibra asset name, e.g. + `Tenant>Capacity>Workspace>my-db`) with a `connectionId` (JDBC connection) to unlock + schema/table/column ingestion, profiling, classification, and sampling for SQL Databases. + Without this, the capability creates only the top-level `Database` asset with no internal + structure. +- `domainIncludeMappings` / `domainExcludeMappings` — route or exclude assets by a + `>`-separated hierarchical path (1–4 segments: Workspace → Artifact → Schema → Table; + wildcards `*` and `?` supported within each segment). Excludes always win over includes. +- `tenantNameOverride` / `capacityNameOverrides` — static name overrides; once set, the + capability stops updating those names from the Microsoft API. + +## Gotchas +- **SQL Database internal structure is not ingested by this capability.** Only the top-level + `Database` asset is created. Configure a `jdbcConnectionsMapping` entry so the + auto-created JDBC Ingestion capability handles schemas, tables, and columns. +- **Warehouse metadata requires `ReadData` permission**, not just Workspace Viewer. The Viewer + role grants only `CONNECT`; `ReadData` (*Read all data using SQL*) must be explicitly granted + in the Fabric portal to allow JDBC `INFORMATION_SCHEMA` queries. +- **Lakehouse table and file access is governed by OneLake security roles**, separate from the + workspace Viewer role. Grant the Service Principal **Read on Tables** (schema/column + discovery via OneLake Table API) and **Read on Files** (file listing via OneLake DFS API) in + the Lakehouse's OneLake security settings — or grant item-level `ReadAll` to bypass role + checks entirely. +- **Mirrored Azure Databricks Catalog tables require two Databricks-side controls**: (1) + `EXTERNAL USE SCHEMA` privilege on each schema in Unity Catalog, and (2) **External data + access enabled on the metastore** (a metastore-level toggle, off by default, requiring + metastore/account admin). If either control is missing, table listing silently returns HTTP + 200 with no tables and column fetches return 403 — ingestion appears to succeed but produces + empty schemas. +- **No admin-consent Power BI/Fabric application permissions** may be registered in Entra for + the Service Principal, even unused ones. Their mere presence breaks service principal + authentication against the Power BI Admin API (used for capacity discovery). +- When both `workspaceNames` and `domainIncludeMappings` are configured, a workspace must + pass **both** filters to be ingested. diff --git a/pkg/skills/files/collibra/etl-integration/references/index.md b/pkg/skills/files/collibra/etl-integration/references/index.md new file mode 100644 index 0000000..fa1d336 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/index.md @@ -0,0 +1,40 @@ +# ETL integration catalog + +Every integration below is an Edge capability configured through the same lifecycle (see +the parent `SKILL.md`): `edge_create_connection` → `edge_create_capability` → +`catalog_etl_get_schema` → `catalog_etl_save_generic_config` → `catalog_etl_start_job` / +`catalog_etl_add_schedule`. + +The **capability typeId** is what you pass to `edge_create_capability` (`typeId`). Always +confirm the exact connection/capability parameters via `edge_list_capability_types` and the +generic-config fields via `catalog_etl_get_schema` — the per-integration notes here are +conceptual, not field-level. + +| Integration | capability typeId | Syncs | +|---|---|---| +| Google Knowledge Catalog / Dataplex | `dataplex-synchronization` | GCP Dataplex / Knowledge Catalog metadata (BigQuery, CloudSQL) — see `dataplex.md` | +| Google Vertex AI | `vertex-synchronization` | Vertex AI models/datasets (GCP) | +| GCP technical lineage | `dataplex-lineage-synchronization` | Technical lineage for GCP | +| Databricks Unity Catalog | `databricks-edge-capability` | Databricks Unity Catalog metadata | +| Databricks lineage | `databricks-lineage` | Technical lineage for Databricks Unity Catalog | +| AWS Bedrock | `bedrock-synchronization` | AWS Bedrock AI models | +| AWS SageMaker | `sagemaker-synchronization` | SageMaker AI (models, etc.) | +| AWS DataZone / SageMaker Unified Studio | `datazone-synchronization` | SageMaker Unified Studio data catalog (Preview) | +| Azure ML | `azureml-synchronization` | Azure Machine Learning | +| Azure AI Foundry | `azure-foundry-synchronization` | Azure AI Foundry | +| Microsoft Fabric | `fabric-synchronization` | Microsoft Fabric | +| Microsoft Purview | `purview-synchronization` | Purview catalog metadata | +| MLflow | `mlflow-synchronization` | MLflow AI experiments/models | +| Sigma | `sigma-synchronization` | Sigma dashboards + lineage | +| ThoughtSpot | `thoughtspot-synchronization` | ThoughtSpot objects + lineage (Preview) | + +**Not listed here / not controllable via chip:** the object-storage crawlers **S3** +(`s3-synchronization`), **ADLS** (`adls-synchronization`), and **GCS** +(`gcs-synchronization`) use a different endpoint/config model and cannot be configured +through the `catalog_etl_*` tools — see "Not controllable via chip (yet)" in `SKILL.md`. + +Notes: +- ThoughtSpot's manifest uses a generated typeId; confirm the live value with + `edge_list_capability_types` if `edge_create_capability` rejects the id above. +- SAP capabilities (AI Core, Datasphere, Data Products, S/4HANA) are out of scope for this + skill — they're owned by the JDBC Ingestion team. diff --git a/pkg/skills/files/collibra/etl-integration/references/mlflow.md b/pkg/skills/files/collibra/etl-integration/references/mlflow.md new file mode 100644 index 0000000..63c4b31 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/mlflow.md @@ -0,0 +1,24 @@ +# MLflow AI (`mlflow-synchronization`) + +Extracts metadata from an MLflow model registry and ingests `MlflowAIModel` assets into a +Collibra Technology Asset Domain. `MlflowAIModel` is a subtype of `AIModel` in the Collibra +AI Governance operating model; the capability creates one asset per registered model (latest +READY version only). + +**Docs:** [About the MLflow AI integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/MLfLowAI/co_about-mlflow.htm) + +**Connection type:** `generic` (HTTP-based). Auth methods (set on `edge_create_connection`): +- **Basic Authentication** — `username` and `password` credentials against the MLflow tracking server URL. +- **OAuth (Client Credentials)** — `tokenUrl`, `clientId`, and `clientSecret`; compatible with Databricks OAuth (M2M). + +**Workflows:** single inbound workflow `main` (default — omit `workflow`). No outbound flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — UUID of the Technology Asset Domain where `MlflowAIModel` assets are created or updated. +- `customAiMetricsMappings` (optional) — array of `{ from, to }` objects mapping an MLflow metric key (e.g., `f1_score`) to a Collibra attribute type UUID. Extends the four standard auto-mapped metrics (`accuracy`, `precision_score`, `mae`, `mse`). A custom mapping entry can also override a standard metric, storing the value in both the standard attribute and the custom one. + +## Gotchas +- Only the **latest READY version** of each registered model is ingested. Models with no `READY` versions are silently skipped — no asset is created or updated. +- If a model version has no associated `run_id`, or the training run was deleted (MLflow returns 404 on `/runs/get`), all metric attributes are silently omitted for that asset. +- Every attribute type referenced in `customAiMetricsMappings` must already be **assigned to the `MlflowAIModel` asset type** (`00000000-0000-0000-0000-000000031410`) in Collibra before the sync runs. Validation happens at startup; a missing assignment aborts the entire synchronization. +- The four standard metric keys (`accuracy`, `precision_score`, `mae`, `mse`) are hard-coded. If the MLflow run uses different key names (e.g., `acc` instead of `accuracy`), the standard attributes will be empty — use `customAiMetricsMappings` to remap them. diff --git a/pkg/skills/files/collibra/etl-integration/references/purview.md b/pkg/skills/files/collibra/etl-integration/references/purview.md new file mode 100644 index 0000000..1bc8fd7 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/purview.md @@ -0,0 +1,37 @@ +# Microsoft Purview (`purview-synchronization`) + +Extracts schema metadata from Azure SQL Server databases cataloged in Microsoft Purview (servers, +databases, schemas, tables, and columns) and ingests it into DGC. `capability-type` (the `typeId` +for `edge_create_capability`): **`purview-synchronization`**. + +**Docs:** No standalone Collibra product page documents this `purview-synchronization` (Azure SQL Server via Purview) capability. In the public docs, "Purview" appears only as a *synchronization source within the ADLS integration* — a different capability — so do not treat the ADLS docs as authoritative for this integration. + +**Connection type:** Azure. Auth method (set on `edge_create_connection`): +- **Service Principal** — `clientId`, `clientSecret`, and `tenantId` from an Azure AD app + registration. + +**Workflows:** single inbound workflow `main` (default — omit `workflow`). No outbound flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `dataSource` (required) — currently only `MSSQL` (Azure SQL Server); defaults to `MSSQL`. +- `system` (required) — target System asset in DGC to which databases are linked. +- `community` (required) — default Collibra community; sub-domains for databases and schemas are + auto-created here when no explicit domain mapping matches. +- `domainIncludeMappings` (optional) — array of `{from, to}` objects. `from` is a + `Server > Database > Schema` glob path (supports `*` and `?` wildcards); `to` is the target + domain. Omitting `to` auto-creates domains inside the default community. +- `domainExcludeMappings` (optional) — array of strings in `Server > Database > Schema` format; + matched paths are skipped even if they match an include rule. + +## Gotchas +- This integration reads from Purview's catalog — it does not scan Azure SQL directly. The Azure + SQL data sources must have already been registered and scanned inside Purview before running the + capability, otherwise no assets will be found. +- The service principal used in the Azure connection needs **Data Reader** and **Data Source Admin** + roles assigned in the Purview account (set in Purview's data-map collections permissions, not in + Azure IAM alone). +- The `purview-account` capability parameter (the short Purview account name, not a URL) is set at + capability creation time and is separate from the generic config. +- Domain mapping paths must be a triplet `Server > Database > Schema`. Paths with fewer or more + levels fail validation at run time with a descriptive error. +- Exclude mappings take priority over include mappings. diff --git a/pkg/skills/files/collibra/etl-integration/references/sagemaker.md b/pkg/skills/files/collibra/etl-integration/references/sagemaker.md new file mode 100644 index 0000000..7982075 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/sagemaker.md @@ -0,0 +1,26 @@ +# AWS SageMaker AI (`sagemaker-synchronization`) + +Extracts ML model metadata from Amazon SageMaker AI and ingests it into DGC as a structured AI asset hierarchy: AI Base Models → AI Model Versions → AI Model Deployments → AI Endpoints + AI Monitors. The `typeId` for `edge_create_capability` is **`sagemaker-synchronization`**. + +**Docs:** [About the AWS SageMaker AI integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/AWSSageMaker/co_about-aws-sagemaker-ai.htm) + +**Connection type:** AWS. Auth methods (set on `edge_create_connection`): +- **IAM** — programmatic user; requires `aws_access_key_id` and `aws_secret_access_key`. +- **EC2** — instance role; no explicit credentials needed (suitable when the Edge site runs on an EC2 instance with an attached IAM role). + +**Workflows:** single inbound workflow `main` (omit the `workflow` argument). No outbound flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — target Collibra Domain where all AI assets are created (AI Base Models, AI Model Versions, AI Model Deployments, AI Endpoints, AI Monitors). +- `regions` (optional) — list of AWS region identifiers to scope ingestion; omit to scan all SageMaker-enabled regions automatically. +- `ingestDeploymentOutput` (optional, default `false`) — when enabled, endpoint output paths from each endpoint configuration are ingested as `FileContainer` and `File` assets linked to the corresponding `AIModelDeployment`. +- `customAiMetricsMappings` (optional) — maps SageMaker metric key names (supports nested JSON lookup) to DGC attribute type UUIDs on the AI model asset. Useful for ingesting non-standard or user-defined model metrics beyond the built-in accuracy/precision/MSE/MAE fields. + +## Gotchas +- Only model packages with `ModelApprovalStatus = Approved` are ingested. Non-approved versions are silently skipped — if model versions are missing, check their approval status in SageMaker. +- The ETL path is selected automatically at runtime based on DGC version: **≥ 2026.03** produces the full AI asset hierarchy (Base Model → Version → Deployment → Endpoint + Monitor); **< 2026.03** produces a single legacy `AWSSagemakerAIModel` asset per model package group (latest approved version only). No configuration is required or possible. +- The IAM user (or role) needs separate permission sets for model ingestion (`sagemaker:ListModelPackageGroups`, `DescribeModelPackageGroup`, `ListModelPackages`, `DescribeModelPackage`), endpoint/deployment ingestion (`sagemaker:ListEndpoints`, `DescribeEndpoint`, `DescribeEndpointConfig`, `DescribeModel`), and monitoring ingestion (`sagemaker:ListMonitoringSchedules`, `DescribeMonitoringSchedule`, `ListMonitoringAlerts`). Missing any group silently drops that category of assets. +- If a deployment variant cannot be linked to an AI Model Version (by package ARN or model data URL), it is still ingested as a standalone deployment with no version relation. The sync report records the failure, but no error is raised. +- An AI Endpoint asset is only created if at least one of its deployment variants was successfully resolved; endpoints with zero resolved variants are dropped. +- `customAiMetricsMappings.from` performs a recursive key search across the metrics JSON. If the found value is a complex object containing a `"value"` key, that inner value is extracted; otherwise the entire object is serialized to a string. Map leaf metric keys (e.g., `recall`, `accuracy`) rather than parent objects to get clean scalar values. +- The `AIMonitor.Url` attribute is only populated on DGC ≥ 2026.04. The `InitiatingUserInSource` attribute on AI assets requires DGC ≥ 2026.07. diff --git a/pkg/skills/files/collibra/etl-integration/references/sigma.md b/pkg/skills/files/collibra/etl-integration/references/sigma.md new file mode 100644 index 0000000..f21d736 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/sigma.md @@ -0,0 +1,23 @@ +# Sigma Computing (`sigma-synchronization`) + +Extracts metadata from Sigma Computing (workspaces, workbooks, datasets, data models, connections, and their columns/elements) and ingests it into DGC. `capability-type` (the `typeId` for `edge_create_capability`): **`sigma-synchronization`**. + +**Docs:** [About Sigma integration](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/Sigma/co_about-sigma-integration.htm) + +**Connection type:** Sigma (OAuth 2.0 client credentials). Auth parameters (set on `edge_create_connection`): +- `clientId` — OAuth client ID from Sigma Admin Portal → Developer Access. +- `clientSecret` — OAuth client secret (shown only once at creation; store it safely). +- `url` — Region-specific Sigma REST API base URL (e.g. `https://aws-api.sigmacomputing.com`; also `gcp-api`, `azure-api` variants). +- `tenantUrl` — Your Sigma tenant URL (e.g. `https://app.sigmacomputing.com/`); used to build asset `Url` attributes. + +**Workflows:** inbound is `main` (default — omit `workflow`). No outbound (DGC → Sigma) flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — UUID of the DGC domain where all ingested assets will land. +- `workspaces` (optional) — array of workspace IDs or case-insensitive workspace names to limit scope. Empty = all workspaces the OAuth client can see. The catalog UI pre-populates this field as a dropdown via an automatic lightweight pre-flight call. + +## Gotchas +- The OAuth client must have **Admin** or **Creator** account type in Sigma. A Member-level client cannot enumerate org-wide content reliably. +- Admin account type alone lets the capability list a workspace and create the workspace asset, but **not** walk its contents (folders, workbooks, datasets, data models). Those workspaces appear in the sync report as "could not be synchronized." Fix: add the OAuth client as a workspace member with at least Viewer permissions. +- Setting `workspace-ids` on the capability **and** `workspaces` in the catalog config simultaneously throws `InvalidParametersException` at startup. Use only one. +- Workspace filter entries that match neither a workspace ID nor a workspace name are tracked as failures. If all configured entries fail, the job status becomes `FAILED`; partial failures yield `COMPLETED_WITH_ERRORS`. diff --git a/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md b/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md new file mode 100644 index 0000000..d2f073c --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md @@ -0,0 +1,37 @@ +# ThoughtSpot Synchronization (`thoughtspot-synchronization`) + +Extracts metadata from a ThoughtSpot cloud tenant — Tenant, Orgs, Connections, Liveboards, Answers, Answer Visualizations, Data Models (Worksheets), Data Entities (Tables/Views), and Data Attributes (columns) — and ingests it into a Collibra DGC domain. A separate lineage workflow additionally submits column-level lineage edges to Collibra TechLin. The `capability-type` (the `typeId` for `edge_create_capability`) is **`thoughtspot-synchronization`** (confirmed in `manifest-template.yaml`; the published `manifest.yaml` uses the `` build-time placeholder, so the resolved value is the same string). + +**Docs:** [About ThoughtSpot integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/CollibraDataLineage/DataSources/ThoughtSpot/co_about-thoughtspot-integration.htm) + +**Connection type:** Generic. Auth method (set on `edge_create_connection`): +- **Username / Password** — ThoughtSpot username (`userName`), password (`password`), and tenant base URL (`url`, e.g. `https://customer.thoughtspot.cloud`). The capability authenticates via the ThoughtSpot REST API v2 full-token endpoint and re-authenticates per org to obtain an org-scoped bearer token. + +**Workflows:** The integration exposes two runnable tabs in the catalog Integration Configuration page: +- **Metadata Inbound** (`default` workflow, `-DstartMode=sync`) — inbound; fetches all ThoughtSpot entity types listed above and creates Catalog assets in the configured DGC domain. +- **ThoughtSpot Lineage** (`lineage` workflow, `-DstartMode=lineage`, Preview) — submits column-level and asset-level lineage relations to Collibra TechLin. Requires the **Metadata Inbound** tab to be configured first. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) + +**Metadata Inbound tab:** +- `domain` (required) — UUID of the DGC domain where ThoughtSpot assets will be created. +- `orgs` (optional) — multi-select list of ThoughtSpot org IDs or names to include. Leave empty to sync all orgs the credentials can see. + +**ThoughtSpot Lineage tab:** +- `domain` (required) — UUID of the DGC domain that owns the lineage source. +- `sourceId` (required) — stable string identifier used to construct the lin-sdk source name (`ThoughtSpot--`). Changing this across runs creates a new parallel lineage source instead of updating the existing one. +- `orgs` (optional) — same org-filter semantics as the sync tab; should be aligned with the sync configuration to avoid partial graphs. + +The lineage capability also requires these **install-time** run-parameters (not in the generic config): `targetDgc` (DGC URL), `username` and `password` (Collibra DGC credentials for lin-sdk — distinct from the ThoughtSpot connection credentials). Optionally `customParameters.techlinHost` + `customParameters.techlinKey` if bypassing DGC-routed lineage submission. + +## Gotchas + +- **Run sync before lineage.** Lineage edges stitch onto assets the sync flow has already created. Running lineage against an empty domain produces a disconnected gray graph in TechLin with no errors. +- **Do not set `org-ids` at the capability level and `orgs` in the generic config at the same time** — the capability throws `InvalidParametersException` at startup. Use one or the other. +- **`sourceId` must be stable.** Changing it between lineage runs creates a new lin-sdk source instead of overwriting the previous one, resulting in duplicate lineage graphs. +- **Lineage is all-or-nothing per run.** If any org's metadata fetch fails during the lineage workflow, the entire job aborts before submitting any edges to TechLin. The sync workflow does not share this property. +- **Data Entities are silently dropped** if the capability cannot resolve their ThoughtSpot Connection (by `dataSourceId` or by name). Their columns are dropped too. This happens when the Connection is not visible to the service user or when multiple connections share a name and the fallback name-match is ambiguous. +- **Private / embed ThoughtSpot deployments** (non-`*.thoughtspot.cloud` hosts) get an incorrect `Url` attribute on the Tenant asset — the synchronization itself still works because API calls use the user-supplied connection URL. +- **Only TABLE-type visualizations** produce per-column Data Attribute assets. CHART, HEADLINE, and other viz types are not expanded into attributes, and the lineage flow cannot draw column-level edges into them. +- **The `defaultStatus` and `cloudIngestionJobId` run-parameters** are declared in the manifest but are not read by the current code — setting them has no effect. +- **`save-input-metadata=true`** (install-time flag) records all source metadata in a downloadable DGC file. Use only when requested by Collibra Support — the file can be large and contains source-system content. diff --git a/pkg/skills/files/collibra/etl-integration/references/vertex.md b/pkg/skills/files/collibra/etl-integration/references/vertex.md new file mode 100644 index 0000000..a5779f0 --- /dev/null +++ b/pkg/skills/files/collibra/etl-integration/references/vertex.md @@ -0,0 +1,38 @@ +# Google Vertex AI (`vertex-synchronization`) + +Extracts AI/ML model metadata from Google Vertex AI (Model Registry, endpoints, +deployments, monitors, reasoning-engine agents, and referenced Model Garden +foundation models) and ingests it into DGC. `capability-type` (the `typeId` for +`edge_create_capability`): **`vertex-synchronization`**. + +**Docs:** [About the Google Vertex AI integration via Edge](https://productresources.collibra.com/docs/collibra/latest/Content/Catalog/IntegrateAIModels/GoogleVertexAI/co_about-vertex-ai.htm) + +**Connection type:** GCP. Auth methods (set on `edge_create_connection`): +- **Service Account** — full JSON key (`gcp_service_account_credentials_json`). +- **Workload Identity Federation (WIF)** — federated identity; token/URL for file-based WIF. +- **WIF using GKE** — when the Edge site runs in GKE; uses the site's federated identity, credentials left blank. + +**Workflows:** inbound only, `main` (default — omit `workflow`). There is no outbound +(DGC → Vertex) flow. + +## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) +- `domain` (required) — target domain for the ingested assets. +- `projects` — GCP Project IDs to scan; if empty, auto-discovers every project the + service account can access. +- `locations` — regions to search; if empty, searches all locations where Vertex AI is available. +- `ingestAiInputAndOutput` (default true) — also ingest training-data File/File Container + assets from GCS and BigQuery prediction-logging output tables; when false, only model + metadata is ingested. +- `systems` — Collibra System assets to search for existing BigQuery output tables before + creating new ones (dedup); only applies when `ingestAiInputAndOutput` is enabled. +- `customLabelMappings` / `customAiMetricsMappings` — map custom Vertex labels/metrics to + AI Model attribute types. + +## Gotchas +- Vendor foundation models (Gemini, Claude, Llama, Mistral, etc.) are ingested **on demand + only** when a customer model references one via `baseModelSource.modelGardenSource`; the + Model Garden catalogue is never proactively scanned. +- If `getPublisherModel` fails (permission/not-found/unavailable), the vendor parent asset + and its parent-link relation are skipped but the sync continues. +- GCS File assets and BigQuery output tables are only produced when `ingestAiInputAndOutput` + is enabled — extra permissions (`storage.objects.*`, `bigquery.tables.get`) are then required. diff --git a/pkg/skills/files/collibra/index/SKILL.md b/pkg/skills/files/collibra/index/SKILL.md index a2f9b15..c5719b3 100644 --- a/pkg/skills/files/collibra/index/SKILL.md +++ b/pkg/skills/files/collibra/index/SKILL.md @@ -1,6 +1,6 @@ --- description: Navigator for chip's Collibra skills. Start here when unsure which skill applies. -related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/jdbc-ingestion +related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/jdbc-ingestion, collibra/etl-integration --- # Collibra skills — navigator @@ -20,6 +20,7 @@ must be bridged to another, and which permissions are required. | Register a table (and its dimension tables) as a Collibra Data Product with ports | `collibra/data-product-create` | | Generate governed YAML context (semantic blueprints, metric definitions) for an asset | `collibra/context` | | Set up an Edge JDBC connection/capability and run jdbc-ingestion | `collibra/jdbc-ingestion` | +| Set up any metadata-sync ETL integration (Dataplex, Databricks UC, Purview, Sigma, …) — connection, capability, generic config, schedule/run | `collibra/etl-integration` | If a task is a single tool call with no chaining (e.g. `get_asset_details` by UUID, `list_asset_types`, `pull_data_contract_manifest`), no skill is needed — the tool's own diff --git a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md index 942a63a..78f4b9d 100644 --- a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md +++ b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md @@ -10,9 +10,26 @@ Setting up jdbc-ingestion is a fixed sequence across two separate systems: Edge No single tool does the whole thing — skipping a step or calling tools out of order is the most common way this goes wrong. +## First: confirm this is the right ingestion path + +Many data sources support **more than one** ingestion path — most commonly a native +**ETL integration** (`collibra/etl-integration`, e.g. Databricks Unity Catalog, Dataplex, +Purview, Sigma) *and* generic **JDBC** (this skill). The native path is usually richer +(lineage, finer metadata) and preferred where it exists. + +**Whenever the user asks to create an integration, before doing anything else, determine +which ingestion type it should be** — don't assume JDBC just because this skill is loaded: + +1. Identify the data source. +2. Check whether more than one path supports it (consult `collibra/etl-integration`'s + `references/index.md` for the native-ETL data sources). +3. **If more than one path is possible, ask the user which they want** before starting. + Only proceed with this JDBC skill once the user has chosen JDBC (or JDBC is the only + supported path). + ## The full sequence -1. **`list_edge_sites`** — get the `edgeSiteId` you'll use everywhere below. +1. **`edge_list_sites`** — get the `edgeSiteId` you'll use everywhere below. 2. **`get_data_source_setup_guide`** (pass the data source name, e.g. `"Snowflake"`) — before guessing any connection parameter. Returns the driver class, connection string format, the Marketplace URL for the driver jar, and per-auth-method @@ -24,15 +41,15 @@ the most common way this goes wrong. `contentBase64` (most JDBC drivers are tens of MB) and this chip instance doesn't have `filePath` enabled, stop and guide the user to create the connection manually in the Edge UI instead — see "Large driver files" below. -4. **`create_connection`** — `typeId: "Generic"` (not a vendor-specific connection +4. **`edge_create_connection`** — `typeId: "Generic"` (not a vendor-specific connection type — jdbc-ingestion requires the Generic JDBC connection type). Fixed manifest parameters (`driver-class`, `connection-string`, `driver-jar` as the `upload_file` URI) go in `parameters`; open-ended vendor properties (Snowflake's Role, Warehouse, User, private key file, etc.) go in `additionalProperties`, not `parameters`. 5. **`test_connection`** — verify it actually connects before building on top of it. Pass `timeoutSec` for a synchronous result, or poll the returned `jobId` with - `get_job_status`. -6. **`create_capability`** — `typeId: "jdbc-ingestion"`, `parameters.connection` set to + `edge_get_job_status`. +6. **`edge_create_capability`** — `typeId: "jdbc-ingestion"`, `parameters.connection` set to the connection id from step 4. 7. Catalog scaffolding, only if it doesn't already exist: **`create_community`** → **`find_domain_types`** (look up e.g. "Physical Data Dictionary") → @@ -48,18 +65,18 @@ the most common way this goes wrong. underlying async refresh hasn't completed yet (cold-start data source connections can take 30–60s) — just call it again; it's idempotent. 9. **`start_ingestion`** — triggers the actual sync using the database id from step 8. -10. **`get_catalog_job_status`** — poll the job id `start_ingestion` returned. A +10. **`get_job_status`** — poll the job id `start_ingestion` returned. A 202/success from `start_ingestion` only means the job was accepted, not that ingestion finished — always poll to confirm completion. ## Two job-id spaces — do not cross them -- **Edge-site jobs** (from `test_connection`) → poll with **`get_job_status`**. -- **DGC catalog jobs** (from `start_ingestion`) → poll with **`get_catalog_job_status`**. +- **Edge-site jobs** (from `test_connection`) → poll with **`edge_get_job_status`**. +- **DGC jobs** (from `start_ingestion`) → poll with **`get_job_status`**. Passing one tool's job id to the other will fail or silently poll the wrong resource. `start_ingestion`'s own `Job.id` is a catalog job id — never poll it with -`get_job_status`. +`edge_get_job_status`. ## Do not use a raw "run capability" tool for this @@ -100,8 +117,8 @@ or schema, though you should still confirm the table pattern (`include`/`exclude If a connection with the driver already exists (most commonly because the driver file was too large to upload and the user created it manually via the Edge UI), use -**`find_connections`** (by name, optionally scoped to `edgeSiteId`) to get its id and -skip straight to step 6 — do not call `create_connection` again for it. +**`edge_find_connections`** (by name, optionally scoped to `edgeSiteId`) to get its id and +skip straight to step 6 — do not call `edge_create_connection` again for it. ## Large driver files @@ -114,7 +131,7 @@ file is available: guide the user to create the whole connection manually via th UI (Settings > Edge > site > Connections > New), giving them the exact field values (driver class, connection string — from `get_data_source_setup_guide`) so they only need to pick the file and click through. Once they confirm it's saved, use -`find_connections` to pick it up and continue from step 6. +`edge_find_connections` to pick it up and continue from step 6. ## Hard rules @@ -122,7 +139,7 @@ need to pick the file and click through. Once they confirm it's saved, use Central, not a GitHub release, not a URL the user pastes that isn't the Marketplace listing. If in doubt, ask the user to download the jar from the URL `get_data_source_setup_guide` returns and hand you the file. -2. **`create_connection`'s `typeId` for jdbc-ingestion is `"Generic"`,** never a +2. **`edge_create_connection`'s `typeId` for jdbc-ingestion is `"Generic"`,** never a vendor-specific connection type — jdbc-ingestion cannot use those. 3. **Always poll after `start_ingestion`.** A successful call only means the job was accepted. diff --git a/pkg/tools/catalog_generic_add_schedule/tool.go b/pkg/tools/catalog_etl_add_schedule/tool.go similarity index 89% rename from pkg/tools/catalog_generic_add_schedule/tool.go rename to pkg/tools/catalog_etl_add_schedule/tool.go index 3942f07..d52d600 100644 --- a/pkg/tools/catalog_generic_add_schedule/tool.go +++ b/pkg/tools/catalog_etl_add_schedule/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_add_schedule +package catalog_etl_add_schedule import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` CronExpression string `json:"cronExpression" jsonschema:"the cron expression for the schedule (e.g. '0 2 * * *')"` CronTimeZone string `json:"cronTimeZone" jsonschema:"the time zone for the cron schedule (e.g. 'UTC', 'America/New_York')"` CronJson string `json:"cronJson,omitempty" jsonschema:"optional JSON representation of the cron schedule"` @@ -27,8 +27,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_add_schedule", - Description: "Adds a synchronization schedule to a GENERIC integration instance. Requires a cron expression and time zone.", + Name: "catalog_etl_add_schedule", + Description: "Adds a synchronization schedule to an ETL integration. Requires a cron expression and time zone.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/catalog_generic_add_schedule/tool_test.go b/pkg/tools/catalog_etl_add_schedule/tool_test.go similarity index 94% rename from pkg/tools/catalog_generic_add_schedule/tool_test.go rename to pkg/tools/catalog_etl_add_schedule/tool_test.go index 094d2f7..e41abe8 100644 --- a/pkg/tools/catalog_generic_add_schedule/tool_test.go +++ b/pkg/tools/catalog_etl_add_schedule/tool_test.go @@ -1,4 +1,4 @@ -package catalog_generic_add_schedule_test +package catalog_etl_add_schedule_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_add_schedule" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_add_schedule" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_cancel_job/tool.go b/pkg/tools/catalog_etl_cancel_job/tool.go similarity index 90% rename from pkg/tools/catalog_generic_cancel_job/tool.go rename to pkg/tools/catalog_etl_cancel_job/tool.go index eb1abe9..033ae58 100644 --- a/pkg/tools/catalog_generic_cancel_job/tool.go +++ b/pkg/tools/catalog_etl_cancel_job/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_cancel_job +package catalog_etl_cancel_job import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` } @@ -23,8 +23,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_cancel_job", - Description: "Cancels the currently running synchronization job for a GENERIC integration instance.", + Name: "catalog_etl_cancel_job", + Description: "Cancels the currently running synchronization job for an ETL integration.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, diff --git a/pkg/tools/catalog_generic_cancel_job/tool_test.go b/pkg/tools/catalog_etl_cancel_job/tool_test.go similarity index 93% rename from pkg/tools/catalog_generic_cancel_job/tool_test.go rename to pkg/tools/catalog_etl_cancel_job/tool_test.go index 8065bc4..2b95c95 100644 --- a/pkg/tools/catalog_generic_cancel_job/tool_test.go +++ b/pkg/tools/catalog_etl_cancel_job/tool_test.go @@ -1,11 +1,11 @@ -package catalog_generic_cancel_job_test +package catalog_etl_cancel_job_test import ( "net/http" "net/http/httptest" "testing" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_cancel_job" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_cancel_job" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_delete_config/tool.go b/pkg/tools/catalog_etl_delete_generic_config/tool.go similarity index 84% rename from pkg/tools/catalog_generic_delete_config/tool.go rename to pkg/tools/catalog_etl_delete_generic_config/tool.go index 6d3b62d..801d134 100644 --- a/pkg/tools/catalog_generic_delete_config/tool.go +++ b/pkg/tools/catalog_etl_delete_generic_config/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_delete_config +package catalog_etl_delete_generic_config import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance whose configuration to delete"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration whose configuration to delete"` } type Output struct { @@ -22,8 +22,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_delete_config", - Description: "Deletes the configuration for a GENERIC integration instance.", + Name: "catalog_etl_delete_generic_config", + Description: "Deletes the configuration for an ETL integration.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, diff --git a/pkg/tools/catalog_generic_delete_config/tool_test.go b/pkg/tools/catalog_etl_delete_generic_config/tool_test.go similarity index 93% rename from pkg/tools/catalog_generic_delete_config/tool_test.go rename to pkg/tools/catalog_etl_delete_generic_config/tool_test.go index d09e5b3..cf478a5 100644 --- a/pkg/tools/catalog_generic_delete_config/tool_test.go +++ b/pkg/tools/catalog_etl_delete_generic_config/tool_test.go @@ -1,11 +1,11 @@ -package catalog_generic_delete_config_test +package catalog_etl_delete_generic_config_test import ( "net/http" "net/http/httptest" "testing" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_delete_config" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_delete_generic_config" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_delete_schedule/tool.go b/pkg/tools/catalog_etl_delete_schedule/tool.go similarity index 86% rename from pkg/tools/catalog_generic_delete_schedule/tool.go rename to pkg/tools/catalog_etl_delete_schedule/tool.go index faaa67a..7b2ced6 100644 --- a/pkg/tools/catalog_generic_delete_schedule/tool.go +++ b/pkg/tools/catalog_etl_delete_schedule/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_delete_schedule +package catalog_etl_delete_schedule import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` } @@ -23,8 +23,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_delete_schedule", - Description: "Deletes the synchronization schedule from a GENERIC integration instance.", + Name: "catalog_etl_delete_schedule", + Description: "Deletes the synchronization schedule from an ETL integration.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true), IdempotentHint: true}, diff --git a/pkg/tools/catalog_generic_delete_schedule/tool_test.go b/pkg/tools/catalog_etl_delete_schedule/tool_test.go similarity index 93% rename from pkg/tools/catalog_generic_delete_schedule/tool_test.go rename to pkg/tools/catalog_etl_delete_schedule/tool_test.go index 781e585..feb1ab8 100644 --- a/pkg/tools/catalog_generic_delete_schedule/tool_test.go +++ b/pkg/tools/catalog_etl_delete_schedule/tool_test.go @@ -1,11 +1,11 @@ -package catalog_generic_delete_schedule_test +package catalog_etl_delete_schedule_test import ( "net/http" "net/http/httptest" "testing" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_delete_schedule" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_delete_schedule" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_get_all_schedules/tool.go b/pkg/tools/catalog_etl_get_all_schedules/tool.go similarity index 86% rename from pkg/tools/catalog_generic_get_all_schedules/tool.go rename to pkg/tools/catalog_etl_get_all_schedules/tool.go index 6aeb619..902073a 100644 --- a/pkg/tools/catalog_generic_get_all_schedules/tool.go +++ b/pkg/tools/catalog_etl_get_all_schedules/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_get_all_schedules +package catalog_etl_get_all_schedules import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` } type Output struct { @@ -23,8 +23,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_get_all_schedules", - Description: "Retrieves all synchronization schedules for a GENERIC integration instance.", + Name: "catalog_etl_get_all_schedules", + Description: "Retrieves all synchronization schedules for an ETL integration.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/catalog_generic_get_all_schedules/tool_test.go b/pkg/tools/catalog_etl_get_all_schedules/tool_test.go similarity index 93% rename from pkg/tools/catalog_generic_get_all_schedules/tool_test.go rename to pkg/tools/catalog_etl_get_all_schedules/tool_test.go index 6b5f185..4efb62c 100644 --- a/pkg/tools/catalog_generic_get_all_schedules/tool_test.go +++ b/pkg/tools/catalog_etl_get_all_schedules/tool_test.go @@ -1,4 +1,4 @@ -package catalog_generic_get_all_schedules_test +package catalog_etl_get_all_schedules_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_get_all_schedules" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_get_all_schedules" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_get_config/tool.go b/pkg/tools/catalog_etl_get_config/tool.go similarity index 81% rename from pkg/tools/catalog_generic_get_config/tool.go rename to pkg/tools/catalog_etl_get_config/tool.go index 074a937..99c301b 100644 --- a/pkg/tools/catalog_generic_get_config/tool.go +++ b/pkg/tools/catalog_etl_get_config/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_get_config +package catalog_etl_get_config import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` } type Output struct { @@ -23,8 +23,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_get_config", - Description: "Retrieves the configuration for a GENERIC integration instance (e.g. Databricks Unity Catalog, Dataplex). The configuration value is a JSON string specific to the integration type.", + Name: "catalog_etl_get_config", + Description: "Retrieves the configuration for an ETL integration (e.g. Databricks Unity Catalog, Dataplex). The configuration value is a JSON string specific to the integration type.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/catalog_generic_get_config/tool_test.go b/pkg/tools/catalog_etl_get_config/tool_test.go similarity index 94% rename from pkg/tools/catalog_generic_get_config/tool_test.go rename to pkg/tools/catalog_etl_get_config/tool_test.go index c3f0e8e..2c56466 100644 --- a/pkg/tools/catalog_generic_get_config/tool_test.go +++ b/pkg/tools/catalog_etl_get_config/tool_test.go @@ -1,4 +1,4 @@ -package catalog_generic_get_config_test +package catalog_etl_get_config_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_get_config" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_get_config" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_get_schedule/tool.go b/pkg/tools/catalog_etl_get_schedule/tool.go similarity index 82% rename from pkg/tools/catalog_generic_get_schedule/tool.go rename to pkg/tools/catalog_etl_get_schedule/tool.go index f9386d4..26bcdd7 100644 --- a/pkg/tools/catalog_generic_get_schedule/tool.go +++ b/pkg/tools/catalog_etl_get_schedule/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_get_schedule +package catalog_etl_get_schedule import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` } type Output struct { @@ -23,8 +23,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_get_schedule", - Description: "Retrieves the synchronization schedule for a GENERIC integration instance. Returns the cron expression, time zone, and timing of the last/next run.", + Name: "catalog_etl_get_schedule", + Description: "Retrieves the synchronization schedule for an ETL integration. Returns the cron expression, time zone, and timing of the last/next run.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/catalog_generic_get_schedule/tool_test.go b/pkg/tools/catalog_etl_get_schedule/tool_test.go similarity index 94% rename from pkg/tools/catalog_generic_get_schedule/tool_test.go rename to pkg/tools/catalog_etl_get_schedule/tool_test.go index af54493..9f7d23c 100644 --- a/pkg/tools/catalog_generic_get_schedule/tool_test.go +++ b/pkg/tools/catalog_etl_get_schedule/tool_test.go @@ -1,4 +1,4 @@ -package catalog_generic_get_schedule_test +package catalog_etl_get_schedule_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_get_schedule" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_get_schedule" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_get_schema/tool.go b/pkg/tools/catalog_etl_get_schema/tool.go similarity index 82% rename from pkg/tools/catalog_generic_get_schema/tool.go rename to pkg/tools/catalog_etl_get_schema/tool.go index 6e8afea..05ebad1 100644 --- a/pkg/tools/catalog_generic_get_schema/tool.go +++ b/pkg/tools/catalog_etl_get_schema/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_get_schema +package catalog_etl_get_schema import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` } type Output struct { @@ -23,8 +23,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_get_schema", - Description: "Retrieves the data schema for a GENERIC integration instance. The schema describes the valid configuration fields that can be used with catalog_generic_save_config.", + Name: "catalog_etl_get_schema", + Description: "Retrieves the data schema for an ETL integration. The schema describes the valid configuration fields that can be used with catalog_etl_save_generic_config.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/catalog_generic_get_schema/tool_test.go b/pkg/tools/catalog_etl_get_schema/tool_test.go similarity index 94% rename from pkg/tools/catalog_generic_get_schema/tool_test.go rename to pkg/tools/catalog_etl_get_schema/tool_test.go index 5a8f0ce..451f318 100644 --- a/pkg/tools/catalog_generic_get_schema/tool_test.go +++ b/pkg/tools/catalog_etl_get_schema/tool_test.go @@ -1,11 +1,11 @@ -package catalog_generic_get_schema_test +package catalog_etl_get_schema_test import ( "net/http" "net/http/httptest" "testing" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_get_schema" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_get_schema" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_save_config/tool.go b/pkg/tools/catalog_etl_save_generic_config/tool.go similarity index 82% rename from pkg/tools/catalog_generic_save_config/tool.go rename to pkg/tools/catalog_etl_save_generic_config/tool.go index 1864c40..9e71108 100644 --- a/pkg/tools/catalog_generic_save_config/tool.go +++ b/pkg/tools/catalog_etl_save_generic_config/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_save_config +package catalog_etl_save_generic_config import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` Configuration string `json:"configuration" jsonschema:"the configuration JSON string to save"` } @@ -24,8 +24,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_save_config", - Description: "Creates or updates the configuration for a GENERIC integration instance. The configuration is a JSON string whose schema depends on the integration type. Use catalog_generic_get_schema to discover valid fields.", + Name: "catalog_etl_save_generic_config", + Description: "Creates or updates the configuration for an ETL integration. The configuration is a JSON string whose schema depends on the integration type. Use catalog_etl_get_schema to discover valid fields.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/catalog_generic_save_config/tool_test.go b/pkg/tools/catalog_etl_save_generic_config/tool_test.go similarity index 94% rename from pkg/tools/catalog_generic_save_config/tool_test.go rename to pkg/tools/catalog_etl_save_generic_config/tool_test.go index 79565a9..d9d159a 100644 --- a/pkg/tools/catalog_generic_save_config/tool_test.go +++ b/pkg/tools/catalog_etl_save_generic_config/tool_test.go @@ -1,4 +1,4 @@ -package catalog_generic_save_config_test +package catalog_etl_save_generic_config_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_save_config" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_save_generic_config" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_start_job/tool.go b/pkg/tools/catalog_etl_start_job/tool.go similarity index 89% rename from pkg/tools/catalog_generic_start_job/tool.go rename to pkg/tools/catalog_etl_start_job/tool.go index 4a35cea..3a64072 100644 --- a/pkg/tools/catalog_generic_start_job/tool.go +++ b/pkg/tools/catalog_etl_start_job/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_start_job +package catalog_etl_start_job import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` CloudIngestionJobId string `json:"cloudIngestionJobId,omitempty" jsonschema:"optional cloud ingestion job ID to resume"` Workflow string `json:"workflow,omitempty" jsonschema:"optional workflow name"` RuntimeArguments string `json:"runtimeArguments,omitempty" jsonschema:"optional runtime arguments as a string"` @@ -26,8 +26,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_start_job", - Description: "Starts a synchronization job for a GENERIC integration instance.", + Name: "catalog_etl_start_job", + Description: "Starts a synchronization job for an ETL integration.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/catalog_generic_start_job/tool_test.go b/pkg/tools/catalog_etl_start_job/tool_test.go similarity index 94% rename from pkg/tools/catalog_generic_start_job/tool_test.go rename to pkg/tools/catalog_etl_start_job/tool_test.go index 3551df6..2d0e40f 100644 --- a/pkg/tools/catalog_generic_start_job/tool_test.go +++ b/pkg/tools/catalog_etl_start_job/tool_test.go @@ -1,4 +1,4 @@ -package catalog_generic_start_job_test +package catalog_etl_start_job_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_start_job" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_start_job" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/catalog_generic_update_schedule/tool.go b/pkg/tools/catalog_etl_update_schedule/tool.go similarity index 88% rename from pkg/tools/catalog_generic_update_schedule/tool.go rename to pkg/tools/catalog_etl_update_schedule/tool.go index 6f42055..51e3e8c 100644 --- a/pkg/tools/catalog_generic_update_schedule/tool.go +++ b/pkg/tools/catalog_etl_update_schedule/tool.go @@ -1,4 +1,4 @@ -package catalog_generic_update_schedule +package catalog_etl_update_schedule import ( "context" @@ -12,7 +12,7 @@ import ( ) type Input struct { - IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the GENERIC integration instance"` + IngestibleId string `json:"ingestibleId" jsonschema:"the UUID of the ETL integration"` CronExpression string `json:"cronExpression" jsonschema:"the updated cron expression (e.g. '0 3 * * *')"` CronTimeZone string `json:"cronTimeZone" jsonschema:"the updated time zone for the cron schedule"` CronJson string `json:"cronJson,omitempty" jsonschema:"optional JSON representation of the cron schedule"` @@ -27,8 +27,8 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "catalog_generic_update_schedule", - Description: "Updates the synchronization schedule for a GENERIC integration instance. Creates the schedule if it does not exist.", + Name: "catalog_etl_update_schedule", + Description: "Updates the synchronization schedule for an ETL integration. Creates the schedule if it does not exist.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/catalog_generic_update_schedule/tool_test.go b/pkg/tools/catalog_etl_update_schedule/tool_test.go similarity index 94% rename from pkg/tools/catalog_generic_update_schedule/tool_test.go rename to pkg/tools/catalog_etl_update_schedule/tool_test.go index 79e8ef6..6e6494e 100644 --- a/pkg/tools/catalog_generic_update_schedule/tool_test.go +++ b/pkg/tools/catalog_etl_update_schedule/tool_test.go @@ -1,4 +1,4 @@ -package catalog_generic_update_schedule_test +package catalog_etl_update_schedule_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/catalog_generic_update_schedule" + tools "github.com/collibra/chip/pkg/tools/catalog_etl_update_schedule" "github.com/collibra/chip/pkg/tools/testutil" ) diff --git a/pkg/tools/configure_database/tool.go b/pkg/tools/configure_database/tool.go index 106cc01..94d8306 100644 --- a/pkg/tools/configure_database/tool.go +++ b/pkg/tools/configure_database/tool.go @@ -42,7 +42,7 @@ const ( ) type Input struct { - EdgeConnectionID string `json:"edgeConnectionId" jsonschema:"UUID of the Edge connection (created via create_connection) to discover a database on."` + EdgeConnectionID string `json:"edgeConnectionId" jsonschema:"UUID of the Edge connection (created via edge_create_connection) to discover a database on."` DatabaseName string `json:"databaseName,omitempty" jsonschema:"Optional. Exact name of the database (catalog) to register, as it appears at the data source. Required if the data source exposes more than one database/catalog through this connection; if there is exactly one, it is selected automatically."` CommunityID string `json:"communityId" jsonschema:"UUID of the community the Database asset (and its automatically created domain) will be created in."` ParentSystemID string `json:"parentSystemId" jsonschema:"UUID of the parent System asset the Database asset will be linked to."` @@ -71,7 +71,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "configure_database", Title: "Configure Database for Ingestion", - Description: "Discovers a database through an Edge connection, registers it as a Database asset, and configures which schemas/tables get synchronized. Prerequisite for start_ingestion. Assumes the target community and parent System asset already exist (create_community/create_domain/create_asset), and that a jdbc-ingestion capability referencing this connection has already been created via create_capability — the discovery/refresh steps here only find data because that capability actually performs the crawl; without one, this fails with a discovery-timeout-shaped error even though the real cause is the missing capability. If more than one database or schema is discovered, or include isn't provided, this returns an error naming the candidates instead of guessing — confirm the database, schemas, and table pattern with the user, do not default to configuring everything.", + Description: "Discovers a database through an Edge connection, registers it as a Database asset, and configures which schemas/tables get synchronized. Prerequisite for start_ingestion. Assumes the target community and parent System asset already exist (create_community/create_domain/create_asset), and that a jdbc-ingestion capability referencing this connection has already been created via edge_create_capability — the discovery/refresh steps here only find data because that capability actually performs the crawl; without one, this fails with a discovery-timeout-shaped error even though the real cause is the missing capability. If more than one database or schema is discovered, or include isn't provided, this returns an error naming the candidates instead of guessing — confirm the database, schemas, and table pattern with the user, do not default to configuring everything.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, @@ -179,7 +179,7 @@ func discoverDatabaseConnection(ctx context.Context, client *http.Client, edgeCo } if len(connections) == 0 { - return nil, fmt.Errorf("no database connections were discovered for edge connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or — more commonly — no capability referencing this connection exists yet: the discovery/refresh only finds data because a jdbc-ingestion capability actually performs the crawl. Verify a capability exists for this connection (create_capability) before retrying", edgeConnectionID, pollAttempts) + return nil, fmt.Errorf("no database connections were discovered for edge connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or — more commonly — no capability referencing this connection exists yet: the discovery/refresh only finds data because a jdbc-ingestion capability actually performs the crawl. Verify a capability exists for this connection (edge_create_capability) before retrying", edgeConnectionID, pollAttempts) } if databaseName != "" { @@ -218,7 +218,7 @@ func discoverSchemaConnections(ctx context.Context, client *http.Client, databas } } - return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or no capability referencing this connection exists yet — verify with create_capability", databaseConnectionID, pollAttempts) + return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or no capability referencing this connection exists yet — verify with edge_create_capability", databaseConnectionID, pollAttempts) } // selectSchemaConnections narrows discovered to the schemas named in schemaNames. An diff --git a/pkg/tools/create_capability/tool.go b/pkg/tools/edge_create_capability/tool.go similarity index 78% rename from pkg/tools/create_capability/tool.go rename to pkg/tools/edge_create_capability/tool.go index eaadb3b..d3610e2 100644 --- a/pkg/tools/create_capability/tool.go +++ b/pkg/tools/edge_create_capability/tool.go @@ -1,7 +1,7 @@ -// Package create_capability implements the create_capability MCP tool: creates or +// Package edge_create_capability implements the edge_create_capability MCP tool: creates or // updates an Edge capability (e.g. jdbc-ingestion or a lineage capability) via the // private Edge capability management API. -package create_capability +package edge_create_capability import ( "context" @@ -18,20 +18,20 @@ type Input struct { CapabilityID string `json:"capabilityId,omitempty" jsonschema:"Optional. UUID of the capability to create or update. If provided, the capability is created with this exact id (or updated if it already exists). If omitted, a new capability is created and the server assigns an id."` Name string `json:"name" jsonschema:"The capability name."` Description string `json:"description,omitempty" jsonschema:"Optional description of the capability."` - TypeID string `json:"typeId" jsonschema:"The id of the capability type (e.g. 'jdbc-ingestion'). Use the list_capability_types tool to discover available types and their expected parameters."` - EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this capability will run. Use the list_edge_sites tool to discover available sites."` - Parameters map[string]any `json:"parameters" jsonschema:"Capability install parameters as defined by the capability type's manifest. For jdbc-ingestion, this includes 'connection' (the id of a connection created via create_connection), 'data-source-type', 'message-mode', and an optional 'other-settings' list of {name, type, value}."` + TypeID string `json:"typeId" jsonschema:"The id of the capability type (e.g. 'jdbc-ingestion'). Use the edge_list_capability_types tool to discover available types and their expected parameters."` + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this capability will run. Use the edge_list_sites tool to discover available sites."` + Parameters map[string]any `json:"parameters" jsonschema:"Capability install parameters as defined by the capability type's manifest. For jdbc-ingestion, this includes 'connection' (the id of a connection created via edge_create_connection), 'data-source-type', 'message-mode', and an optional 'other-settings' list of {name, type, value}."` } type Output struct { Capability *clients.EdgeCapability `json:"capability,omitempty" jsonschema:"The created or updated capability."` - Success bool `json:"success" jsonschema:"Whether the capability was saved successfully."` - Error string `json:"error,omitempty" jsonschema:"Error message if saving the capability failed."` + Success bool `json:"success" jsonschema:"Whether the capability was saved successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if saving the capability failed."` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "create_capability", + Name: "edge_create_capability", Title: "Create or Update Edge Capability", Description: "Creates or updates an Edge capability (e.g. jdbc-ingestion) via the private Edge capability management API. Does not run the capability — use start_ingestion to trigger a jdbc-ingestion run.", Handler: handler(collibraClient), diff --git a/pkg/tools/create_capability/tool_test.go b/pkg/tools/edge_create_capability/tool_test.go similarity index 96% rename from pkg/tools/create_capability/tool_test.go rename to pkg/tools/edge_create_capability/tool_test.go index f183c76..444c648 100644 --- a/pkg/tools/create_capability/tool_test.go +++ b/pkg/tools/edge_create_capability/tool_test.go @@ -1,4 +1,4 @@ -package create_capability_test +package edge_create_capability_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/create_capability" + tools "github.com/collibra/chip/pkg/tools/edge_create_capability" "github.com/collibra/chip/pkg/tools/testutil" "github.com/google/uuid" ) diff --git a/pkg/tools/create_connection/tool.go b/pkg/tools/edge_create_connection/tool.go similarity index 90% rename from pkg/tools/create_connection/tool.go rename to pkg/tools/edge_create_connection/tool.go index 855441f..dbf2f48 100644 --- a/pkg/tools/create_connection/tool.go +++ b/pkg/tools/edge_create_connection/tool.go @@ -1,7 +1,7 @@ -// Package create_connection implements the create_connection MCP tool: creates or +// Package edge_create_connection implements the edge_create_connection MCP tool: creates or // updates an Edge connection (e.g. a JDBC connection backing the jdbc-ingestion // capability) via the private Edge connection management API. -package create_connection +package edge_create_connection import ( "context" @@ -18,13 +18,13 @@ type Input struct { ConnectionID string `json:"connectionId,omitempty" jsonschema:"Optional. UUID of the connection to create or update. If provided, the connection is created with this exact id (or updated if it already exists). If omitted, a new connection is created and the server assigns an id."` Name string `json:"name" jsonschema:"The connection name."` Description string `json:"description,omitempty" jsonschema:"Optional description of the connection."` - TypeID string `json:"typeId" jsonschema:"The id of the connection type (e.g. 'Generic' for a generic JDBC connection, or a vendor-specific type such as a Snowflake connection type). Determines which parameters are expected — use list_capability_types to inspect a type's manifest."` - EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this connection will be valid. Use the list_edge_sites tool to discover available sites."` + TypeID string `json:"typeId" jsonschema:"The id of the connection type (e.g. 'Generic' for a generic JDBC connection, or a vendor-specific type such as a Snowflake connection type). Determines which parameters are expected — use edge_list_capability_types to inspect a type's manifest."` + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site where this connection will be valid. Use the edge_list_sites tool to discover available sites."` VaultID string `json:"vaultId,omitempty" jsonschema:"Optional UUID of the vault to retrieve vault-backed parameters from."` Parameters map[string]any `json:"parameters" jsonschema:"Fixed connection parameters as defined by the connection type's manifest (e.g. driver-class, connection-string for the 'Generic' JDBC connection type). Do NOT put open-ended/vendor-specific properties (e.g. a Snowflake connection's Role, Warehouse, User, Database, or a private key file) here — use additionalProperties instead. If you don't already know a data source's driver class, connection string format, required properties, or verified driver source, call get_data_source_setup_guide first instead of guessing. For 'driver-jar' (and any other FILE-type parameter), the value is an artifact URI from upload_file — see that tool's policy on where driver files may come from."` AdditionalProperties []AdditionalProperty `json:"additionalProperties,omitempty" jsonschema:"Optional. Open-ended, connection-type-defined properties beyond the fixed manifest parameters — e.g. a Snowflake connection (via the 'Generic' JDBC connection type) uses this for Role, Warehouse, User, Database, and private_key_file. Confirmed shape: each entry is injected as {name, type, value, secret} into a single array-valued parameter (see additionalPropertiesKey). For a FILE-type property, first call upload_file and use its returned artifact URI as this entry's value with type='file'."` - AdditionalPropertiesKey string `json:"additionalPropertiesKey,omitempty" jsonschema:"Optional. The manifest parameter name that additionalProperties are grouped under. Defaults to 'connection-properties', which is correct for the 'Generic' JDBC connection type (confirmed live). Other connection types may use a different key (e.g. AWS/GCP connections use 'additional-parameters') — check the type's manifest via list_capability_types if unsure."` + AdditionalPropertiesKey string `json:"additionalPropertiesKey,omitempty" jsonschema:"Optional. The manifest parameter name that additionalProperties are grouped under. Defaults to 'connection-properties', which is correct for the 'Generic' JDBC connection type (confirmed live). Other connection types may use a different key (e.g. AWS/GCP connections use 'additional-parameters') — check the type's manifest via edge_list_capability_types if unsure."` } // AdditionalProperty is one entry of an open-ended, connection-type-defined property @@ -41,13 +41,13 @@ type AdditionalProperty struct { type Output struct { Connection *clients.EdgeConnection `json:"connection,omitempty" jsonschema:"The created or updated connection."` - Success bool `json:"success" jsonschema:"Whether the connection was saved successfully."` - Error string `json:"error,omitempty" jsonschema:"Error message if saving the connection failed."` + Success bool `json:"success" jsonschema:"Whether the connection was saved successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if saving the connection failed."` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "create_connection", + Name: "edge_create_connection", Title: "Create or Update Edge Connection", Description: "Creates or updates an Edge connection (e.g. a JDBC connection for the jdbc-ingestion capability) via the private Edge connection management API. Does not download driver files itself — see upload_file for how driver jars must be sourced (Collibra Marketplace only, not arbitrary URLs).", Handler: handler(collibraClient), diff --git a/pkg/tools/create_connection/tool_test.go b/pkg/tools/edge_create_connection/tool_test.go similarity index 97% rename from pkg/tools/create_connection/tool_test.go rename to pkg/tools/edge_create_connection/tool_test.go index 8213310..cf1dd8a 100644 --- a/pkg/tools/create_connection/tool_test.go +++ b/pkg/tools/edge_create_connection/tool_test.go @@ -1,4 +1,4 @@ -package create_connection_test +package edge_create_connection_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/create_connection" + tools "github.com/collibra/chip/pkg/tools/edge_create_connection" "github.com/collibra/chip/pkg/tools/testutil" "github.com/google/uuid" ) @@ -182,7 +182,7 @@ func TestCreateConnection_AdditionalProperties_CustomKey(t *testing.T) { } } -// create_connection deliberately does not fetch driver files from arbitrary URLs +// edge_create_connection deliberately does not fetch driver files from arbitrary URLs // (Collibra Marketplace requires an authenticated download — see upload_file's // policy doc) — the only supported path is referencing an artifact URI that was // already uploaded via upload_file, passed through like any other parameter. diff --git a/pkg/tools/edge_find_capabilities/tool.go b/pkg/tools/edge_find_capabilities/tool.go index 21ede57..93287f6 100644 --- a/pkg/tools/edge_find_capabilities/tool.go +++ b/pkg/tools/edge_find_capabilities/tool.go @@ -19,8 +19,8 @@ type Input struct { type Output struct { Capabilities []clients.EdgeCapability `json:"capabilities" jsonschema:"list of matching capabilities"` - Count int `json:"count" jsonschema:"number of matching capabilities"` - Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` + Count int `json:"count" jsonschema:"number of matching capabilities"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { @@ -51,4 +51,4 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { } return Output{Capabilities: caps, Count: len(caps)}, nil } -} \ No newline at end of file +} diff --git a/pkg/tools/edge_find_capabilities/tool_test.go b/pkg/tools/edge_find_capabilities/tool_test.go index 015b35a..991485f 100644 --- a/pkg/tools/edge_find_capabilities/tool_test.go +++ b/pkg/tools/edge_find_capabilities/tool_test.go @@ -63,4 +63,4 @@ func TestEdgeFindCapabilitiesInvalidSiteId(t *testing.T) { if err == nil { t.Fatal("expected validation error") } -} \ No newline at end of file +} diff --git a/pkg/tools/find_connections/tool.go b/pkg/tools/edge_find_connections/tool.go similarity index 86% rename from pkg/tools/find_connections/tool.go rename to pkg/tools/edge_find_connections/tool.go index 60291a1..7d780e0 100644 --- a/pkg/tools/find_connections/tool.go +++ b/pkg/tools/edge_find_connections/tool.go @@ -1,9 +1,9 @@ -// Package find_connections implements the find_connections MCP tool: finds Edge +// Package edge_find_connections implements the edge_find_connections MCP tool: finds Edge // connections by name. Primarily for picking up a connection the user created // manually (e.g. via the DGC/Edge UI, when a driver file was too large to upload -// through upload_file's contentBase64) — see create_connection's package doc for the +// through upload_file's contentBase64) — see edge_create_connection's package doc for the // guided-manual-creation flow this supports. -package find_connections +package edge_find_connections import ( "context" @@ -27,9 +27,9 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "find_connections", + Name: "edge_find_connections", Title: "Find Edge Connections", - Description: "Finds Edge connections by name (and optionally edge site). Use this to pick up a connection that already exists — e.g. one the user created manually via the DGC/Edge UI for a driver file too large to pass through upload_file — instead of creating a duplicate with create_connection.", + Description: "Finds Edge connections by name (and optionally edge site). Use this to pick up a connection that already exists — e.g. one the user created manually via the DGC/Edge UI for a driver file too large to pass through upload_file — instead of creating a duplicate with edge_create_connection.", Handler: handler(collibraClient), Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/find_connections/tool_test.go b/pkg/tools/edge_find_connections/tool_test.go similarity index 93% rename from pkg/tools/find_connections/tool_test.go rename to pkg/tools/edge_find_connections/tool_test.go index 6706aad..171e50b 100644 --- a/pkg/tools/find_connections/tool_test.go +++ b/pkg/tools/edge_find_connections/tool_test.go @@ -1,4 +1,4 @@ -package find_connections_test +package edge_find_connections_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/find_connections" + tools "github.com/collibra/chip/pkg/tools/edge_find_connections" "github.com/collibra/chip/pkg/tools/testutil" "github.com/google/uuid" ) diff --git a/pkg/tools/edge_get_capability/tool.go b/pkg/tools/edge_get_capability/tool.go index 5b58f02..76003ad 100644 --- a/pkg/tools/edge_get_capability/tool.go +++ b/pkg/tools/edge_get_capability/tool.go @@ -17,8 +17,8 @@ type Input struct { type Output struct { Capability *clients.EdgeCapability `json:"capability,omitempty" jsonschema:"the capability details"` - Found bool `json:"found" jsonschema:"whether the capability was found"` - Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` + Found bool `json:"found" jsonschema:"whether the capability was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { @@ -42,4 +42,4 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { } return Output{Capability: cap, Found: true}, nil } -} \ No newline at end of file +} diff --git a/pkg/tools/edge_get_capability/tool_test.go b/pkg/tools/edge_get_capability/tool_test.go index 88590ff..4a64c3f 100644 --- a/pkg/tools/edge_get_capability/tool_test.go +++ b/pkg/tools/edge_get_capability/tool_test.go @@ -64,4 +64,4 @@ func TestEdgeGetCapabilityInvalidId(t *testing.T) { if err == nil { t.Fatal("expected validation error") } -} \ No newline at end of file +} diff --git a/pkg/tools/edge_get_connection/tool.go b/pkg/tools/edge_get_connection/tool.go index 0b660e8..1b00a1f 100644 --- a/pkg/tools/edge_get_connection/tool.go +++ b/pkg/tools/edge_get_connection/tool.go @@ -17,8 +17,8 @@ type Input struct { type Output struct { Connection *clients.EdgeConnection `json:"connection,omitempty" jsonschema:"the connection details"` - Found bool `json:"found" jsonschema:"whether the connection was found"` - Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` + Found bool `json:"found" jsonschema:"whether the connection was found"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { diff --git a/pkg/tools/edge_get_job_status/tool.go b/pkg/tools/edge_get_job_status/tool.go new file mode 100644 index 0000000..9bf6456 --- /dev/null +++ b/pkg/tools/edge_get_job_status/tool.go @@ -0,0 +1,49 @@ +// Package edge_get_job_status implements the edge_get_job_status MCP tool: polls the status of +// an Edge job, e.g. one started by test_connection or start_ingestion. +package edge_get_job_status + +import ( + "context" + "net/http" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type Input struct { + JobID string `json:"jobId" jsonschema:"UUID of the job to check, as returned by test_connection or start_ingestion."` +} + +type Output struct { + Status string `json:"status,omitempty" jsonschema:"The job's current status: UNKNOWN, SUBMITTED, DOWNLOADING, SCHEDULED, RUNNING, SUCCEEDED, FAILED, CANCELLED, CANCELLING, CAPABILITY_PROGRESS, CAPABILITY_SUCCEEDED, or CAPABILITY_FAILED. SUCCEEDED/FAILED/CANCELLED/CAPABILITY_SUCCEEDED/CAPABILITY_FAILED are terminal."` + Message string `json:"message,omitempty" jsonschema:"Status message from the job."` + LastUpdatedDateTime string `json:"lastUpdatedDateTime,omitempty" jsonschema:"When the status was last updated."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "edge_get_job_status", + Title: "Get Edge Job Status", + Description: "Gets the current status of an Edge job, e.g. one started by test_connection or start_ingestion.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("jobId", input.JobID); err != nil { + return Output{}, err + } + + status, err := clients.GetJobStatusLog(ctx, collibraClient, input.JobID) + if err != nil { + return Output{}, err + } + + return Output{Status: status.Status, Message: status.Message, LastUpdatedDateTime: status.LastUpdatedDateTime}, nil + } +} diff --git a/pkg/tools/edge_get_job_status/tool_test.go b/pkg/tools/edge_get_job_status/tool_test.go new file mode 100644 index 0000000..c13b73e --- /dev/null +++ b/pkg/tools/edge_get_job_status/tool_test.go @@ -0,0 +1,46 @@ +package edge_get_job_status_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/edge_get_job_status" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func TestGetJobStatus(t *testing.T) { + jobID, _ := uuid.NewUUID() + + handler := http.NewServeMux() + handler.Handle("GET /edge/api/rest/v2/jobs/"+jobID.String()+"/statusLog", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobStatusLog) { + return http.StatusOK, clients.JobStatusLog{ + JobID: jobID.String(), + Status: "CAPABILITY_SUCCEEDED", + Message: "done", + LastUpdatedDateTime: "2026-07-04T10:00:00.000Z", + } + })) + + server := httptest.NewServer(handler) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: jobID.String()}) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if output.Status != "CAPABILITY_SUCCEEDED" { + t.Fatalf("unexpected status: %s", output.Status) + } +} + +func TestGetJobStatus_InvalidJobID(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: "not-a-uuid"}) + if err == nil { + t.Fatalf("expected an error for invalid jobId") + } +} diff --git a/pkg/tools/edge_get_job_status_history/tool.go b/pkg/tools/edge_get_job_status_history/tool.go index ab6cca7..d87c548 100644 --- a/pkg/tools/edge_get_job_status_history/tool.go +++ b/pkg/tools/edge_get_job_status_history/tool.go @@ -17,8 +17,8 @@ type Input struct { type Output struct { History []clients.JobStatusLog `json:"history" jsonschema:"all job status updates in LIFO order (most recent first)"` - Count int `json:"count" jsonschema:"number of status entries returned"` - Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` + Count int `json:"count" jsonschema:"number of status entries returned"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { diff --git a/pkg/tools/edge_list_capabilities/tool.go b/pkg/tools/edge_list_capabilities/tool.go index 71b60a0..b958532 100644 --- a/pkg/tools/edge_list_capabilities/tool.go +++ b/pkg/tools/edge_list_capabilities/tool.go @@ -18,8 +18,8 @@ type Input struct { type Output struct { Capabilities []clients.EdgeCapability `json:"capabilities" jsonschema:"list of Edge capabilities"` - Count int `json:"count" jsonschema:"number of capabilities returned"` - Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` + Count int `json:"count" jsonschema:"number of capabilities returned"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { diff --git a/pkg/tools/list_capability_types/tool.go b/pkg/tools/edge_list_capability_types/tool.go similarity index 90% rename from pkg/tools/list_capability_types/tool.go rename to pkg/tools/edge_list_capability_types/tool.go index a27456f..5798e72 100644 --- a/pkg/tools/list_capability_types/tool.go +++ b/pkg/tools/edge_list_capability_types/tool.go @@ -1,7 +1,7 @@ -// Package list_capability_types implements the list_capability_types MCP tool: a +// Package edge_list_capability_types implements the edge_list_capability_types MCP tool: a // read-only discovery tool for finding valid typeId values (and their manifests) for -// create_capability and create_connection on a given edge site. -package list_capability_types +// edge_create_capability and edge_create_connection on a given edge site. +package edge_list_capability_types import ( "context" @@ -15,7 +15,7 @@ import ( ) type Input struct { - EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site to list available capability and connection types for. Use the list_edge_sites tool to discover available sites."` + EdgeSiteID string `json:"edgeSiteId" jsonschema:"UUID of the edge site to list available capability and connection types for. Use the edge_list_sites tool to discover available sites."` Query string `json:"query,omitempty" jsonschema:"Optional. Case-insensitive substring filter on type id (e.g. 'jdbc', 'snowflake'). Without a query, all type ids are returned WITHOUT their manifest (an edge site can have 80+ types; manifests are large) — call again with a query matching the type you want to see its full manifest with expected parameters."` } @@ -26,9 +26,9 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "list_capability_types", + Name: "edge_list_capability_types", Title: "List Capability and Connection Types", - Description: "Lists the capability and connection types available on an edge site. Without query, returns just the ids (an edge site can have 80+ types, each with a large manifest). Pass query to filter to matching types and get their full manifest describing expected parameters for create_capability/create_connection.", + Description: "Lists the capability and connection types available on an edge site. Without query, returns just the ids (an edge site can have 80+ types, each with a large manifest). Pass query to filter to matching types and get their full manifest describing expected parameters for edge_create_capability/edge_create_connection.", Handler: handler(collibraClient), Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/list_capability_types/tool_test.go b/pkg/tools/edge_list_capability_types/tool_test.go similarity index 97% rename from pkg/tools/list_capability_types/tool_test.go rename to pkg/tools/edge_list_capability_types/tool_test.go index 29f769c..9f93a28 100644 --- a/pkg/tools/list_capability_types/tool_test.go +++ b/pkg/tools/edge_list_capability_types/tool_test.go @@ -1,4 +1,4 @@ -package list_capability_types_test +package edge_list_capability_types_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/list_capability_types" + tools "github.com/collibra/chip/pkg/tools/edge_list_capability_types" "github.com/collibra/chip/pkg/tools/testutil" "github.com/google/uuid" ) diff --git a/pkg/tools/edge_list_connections/tool.go b/pkg/tools/edge_list_connections/tool.go index 01a95cb..a241aa0 100644 --- a/pkg/tools/edge_list_connections/tool.go +++ b/pkg/tools/edge_list_connections/tool.go @@ -13,8 +13,8 @@ type Input struct{} type Output struct { Connections []clients.EdgeConnection `json:"connections" jsonschema:"list of Edge connections"` - Count int `json:"count" jsonschema:"number of connections returned"` - Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` + Count int `json:"count" jsonschema:"number of connections returned"` + Error string `json:"error,omitempty" jsonschema:"error message if the request failed"` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { diff --git a/pkg/tools/list_edge_sites/tool.go b/pkg/tools/edge_list_sites/tool.go similarity index 73% rename from pkg/tools/list_edge_sites/tool.go rename to pkg/tools/edge_list_sites/tool.go index d8cd8d6..c27827c 100644 --- a/pkg/tools/list_edge_sites/tool.go +++ b/pkg/tools/edge_list_sites/tool.go @@ -1,7 +1,7 @@ -// Package list_edge_sites implements the list_edge_sites MCP tool: a read-only -// discovery tool for finding the edgeSiteId needed by create_connection and -// create_capability. -package list_edge_sites +// Package edge_list_sites implements the edge_list_sites MCP tool: a read-only +// discovery tool for finding the edgeSiteId needed by edge_create_connection and +// edge_create_capability. +package edge_list_sites import ( "context" @@ -20,9 +20,9 @@ type Output struct { func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ - Name: "list_edge_sites", + Name: "edge_list_sites", Title: "List Edge Sites", - Description: "Lists available Edge sites, including their id, name, and status. Use this to find the edgeSiteId needed by create_connection and create_capability.", + Description: "Lists available Edge sites, including their id, name, and status. Use this to find the edgeSiteId needed by edge_create_connection and edge_create_capability.", Handler: handler(collibraClient), Permissions: []string{"dgc.edge-view-connections-and-capabilities"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/list_edge_sites/tool_test.go b/pkg/tools/edge_list_sites/tool_test.go similarity index 91% rename from pkg/tools/list_edge_sites/tool_test.go rename to pkg/tools/edge_list_sites/tool_test.go index 93a1d39..2362c25 100644 --- a/pkg/tools/list_edge_sites/tool_test.go +++ b/pkg/tools/edge_list_sites/tool_test.go @@ -1,4 +1,4 @@ -package list_edge_sites_test +package edge_list_sites_test import ( "net/http" @@ -6,7 +6,7 @@ import ( "testing" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/list_edge_sites" + tools "github.com/collibra/chip/pkg/tools/edge_list_sites" "github.com/collibra/chip/pkg/tools/testutil" "github.com/google/uuid" ) diff --git a/pkg/tools/find_domain_types/tool.go b/pkg/tools/find_domain_types/tool.go index a34ac04..1c987aa 100644 --- a/pkg/tools/find_domain_types/tool.go +++ b/pkg/tools/find_domain_types/tool.go @@ -1,7 +1,7 @@ // Package find_domain_types implements the find_domain_types MCP tool: resolves a // domain type's name (e.g. "Physical Data Dictionary") to its UUID, for // create_domain's typeId — there is no other discovery path for this, since domain -// types are not returned by list_capability_types or any Edge API. +// types are not returned by edge_list_capability_types or any Edge API. package find_domain_types import ( diff --git a/pkg/tools/get_catalog_job_status/tool.go b/pkg/tools/get_catalog_job_status/tool.go deleted file mode 100644 index 6b2e35f..0000000 --- a/pkg/tools/get_catalog_job_status/tool.go +++ /dev/null @@ -1,65 +0,0 @@ -// Package get_catalog_job_status implements the get_catalog_job_status MCP tool: -// polls the status of a DGC catalog job, e.g. the one returned by start_ingestion. -// This is distinct from get_job_status, which polls Edge-site jobs (capability runs, -// connection tests) — the two job id spaces are not interchangeable. -package get_catalog_job_status - -import ( - "context" - "net/http" - - "github.com/collibra/chip/pkg/chip" - "github.com/collibra/chip/pkg/clients" - "github.com/collibra/chip/pkg/tools/validation" - "github.com/modelcontextprotocol/go-sdk/mcp" -) - -type Input struct { - JobID string `json:"jobId" jsonschema:"UUID of the DGC catalog job to check, as returned by start_ingestion. Do not pass an Edge job id here (e.g. from test_connection) — use get_job_status for those instead."` -} - -type Output struct { - Name string `json:"name,omitempty" jsonschema:"The job's name."` - Type string `json:"type,omitempty" jsonschema:"The job's type (e.g. 'DELTA_INGESTION')."` - State string `json:"state,omitempty" jsonschema:"The job's current state: WAITING, RUNNING, CANCELING, COMPLETED, CANCELED, or ERROR."` - Result string `json:"result,omitempty" jsonschema:"The job's result once finished: NOT_SET (not finished yet), SUCCESS, COMPLETED_WITH_ERROR, FAILURE, or ABORTED."` - Message string `json:"message,omitempty" jsonschema:"Status message from the job."` - ProgressPercentage int `json:"progressPercentage,omitempty" jsonschema:"Progress percentage, 0-100."` - StartDate string `json:"startDate,omitempty"` - EndDate string `json:"endDate,omitempty" jsonschema:"When the job finished, if it has."` -} - -func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { - return &chip.Tool[Input, Output]{ - Name: "get_catalog_job_status", - Title: "Get Catalog Job Status", - Description: "Gets the current status of a DGC catalog job, e.g. the ingestion job started by start_ingestion. For Edge-site jobs (test_connection, capability runs), use get_job_status instead — the two job id spaces are separate and not interchangeable.", - Handler: handler(collibraClient), - Permissions: []string{}, - Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, - } -} - -func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { - return func(ctx context.Context, input Input) (Output, error) { - if err := validation.UUID("jobId", input.JobID); err != nil { - return Output{}, err - } - - job, err := clients.GetJob(ctx, collibraClient, input.JobID) - if err != nil { - return Output{}, err - } - - return Output{ - Name: job.Name, - Type: job.Type, - State: job.State, - Result: job.Result, - Message: job.Message, - ProgressPercentage: job.ProgressPercentage, - StartDate: job.StartDate, - EndDate: job.EndDate, - }, nil - } -} diff --git a/pkg/tools/get_catalog_job_status/tool_test.go b/pkg/tools/get_catalog_job_status/tool_test.go deleted file mode 100644 index a5840c9..0000000 --- a/pkg/tools/get_catalog_job_status/tool_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package get_catalog_job_status_test - -import ( - "net/http" - "net/http/httptest" - "testing" - - tools "github.com/collibra/chip/pkg/tools/get_catalog_job_status" - "github.com/collibra/chip/pkg/tools/testutil" - "github.com/google/uuid" -) - -func TestGetCatalogJobStatus(t *testing.T) { - jobID, _ := uuid.NewUUID() - - handler := http.NewServeMux() - handler.HandleFunc("GET /rest/jobs/v1/jobs/"+jobID.String(), func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"id":"` + jobID.String() + `","name":"Database Synchronisation of source","type":"DELTA_INGESTION","state":"COMPLETED","result":"SUCCESS","progressPercentage":100}`)) - }) - - server := httptest.NewServer(handler) - defer server.Close() - - client := testutil.NewClient(server) - output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: jobID.String()}) - if err != nil { - t.Fatalf("expected no error, got: %v", err) - } - if output.State != "COMPLETED" || output.Result != "SUCCESS" { - t.Fatalf("unexpected output: %+v", output) - } -} - -func TestGetCatalogJobStatus_InvalidJobID(t *testing.T) { - client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) - _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: "not-a-uuid"}) - if err == nil { - t.Fatalf("expected an error for invalid jobId") - } -} diff --git a/pkg/tools/get_data_source_setup_guide/tool.go b/pkg/tools/get_data_source_setup_guide/tool.go index 15923ac..65505c9 100644 --- a/pkg/tools/get_data_source_setup_guide/tool.go +++ b/pkg/tools/get_data_source_setup_guide/tool.go @@ -1,7 +1,7 @@ // Package get_data_source_setup_guide implements the get_data_source_setup_guide MCP // tool: looks up driver class, JDBC connection string format, and per-auth-method // connection properties for any of Collibra's ~68 supported data sources, so -// create_connection can be filled in correctly without the agent (or the user) +// edge_create_connection can be filled in correctly without the agent (or the user) // needing to already know a given vendor's JDBC driver properties. // // This is a documented gap, not a guaranteed integration: the underlying data comes @@ -30,11 +30,11 @@ type Input struct { type Output struct { DataSourceID string `json:"dataSourceId,omitempty" jsonschema:"The matched data source's internal id (slug)."` DataSourceName string `json:"dataSourceName,omitempty" jsonschema:"The matched data source's display name."` - DriverClassName string `json:"driverClassName,omitempty" jsonschema:"The JDBC driver class name for create_connection's 'driver-class' parameter."` - ConnectionStringFormat string `json:"connectionStringFormat,omitempty" jsonschema:"The JDBC connection string format for create_connection's 'connection-string' parameter, with placeholders (e.g. 'jdbc:snowflake://.snowflakecomputing.com')."` - MarketplaceURL string `json:"marketplaceUrl,omitempty" jsonschema:"Collibra Marketplace listing URL for this data source's JDBC driver, if the driver needs to be downloaded before uploading via create_connection's driverJarUrl/driverJarFilename or the upload_file tool."` + DriverClassName string `json:"driverClassName,omitempty" jsonschema:"The JDBC driver class name for edge_create_connection's 'driver-class' parameter."` + ConnectionStringFormat string `json:"connectionStringFormat,omitempty" jsonschema:"The JDBC connection string format for edge_create_connection's 'connection-string' parameter, with placeholders (e.g. 'jdbc:snowflake://.snowflakecomputing.com')."` + MarketplaceURL string `json:"marketplaceUrl,omitempty" jsonschema:"Collibra Marketplace listing URL for this data source's JDBC driver, if the driver needs to be downloaded before uploading via edge_create_connection's driverJarUrl/driverJarFilename or the upload_file tool."` Prerequisites string `json:"prerequisites,omitempty" jsonschema:"Documented prerequisites before creating this connection."` - AuthMethods []clients.AuthMethodProperties `json:"authMethods,omitempty" jsonschema:"Documented connection properties, grouped by supported authentication method. Pick the method matching what credentials the user has, then pass its properties as create_connection's additionalProperties (name/type/value/secret per entry — secret=true when the property's Value Type here is 'Secret'). A 'File' type property's value must first be uploaded via upload_file or create_connection's driverJarUrl, then referenced by the returned artifact URI."` + AuthMethods []clients.AuthMethodProperties `json:"authMethods,omitempty" jsonschema:"Documented connection properties, grouped by supported authentication method. Pick the method matching what credentials the user has, then pass its properties as edge_create_connection's additionalProperties (name/type/value/secret per entry — secret=true when the property's Value Type here is 'Secret'). A 'File' type property's value must first be uploaded via upload_file or edge_create_connection's driverJarUrl, then referenced by the returned artifact URI."` Matches []clients.DataSourceSlug `json:"matches,omitempty" jsonschema:"Candidate data sources when the lookup was ambiguous or not found. Retry with one of these exact ids."` Success bool `json:"success" jsonschema:"Whether at least some setup information was found."` Error string `json:"error,omitempty" jsonschema:"Details if part or all of the lookup failed. A failure here does not mean the data source is unsupported by Edge — it means this best-effort documentation lookup didn't find it; ask the user for the connection properties directly."` @@ -44,7 +44,7 @@ func NewTool(_ *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "get_data_source_setup_guide", Title: "Get Data Source Connection Setup Guide", - Description: "Looks up driver class, JDBC connection string format, and per-auth-method connection properties for a data source, from Collibra's public documentation (best-effort scrape, not a stable API — see tool source for details). Use before create_connection when the required parameters for a data source (e.g. Snowflake's Role/Warehouse/private key) aren't already known.", + Description: "Looks up driver class, JDBC connection string format, and per-auth-method connection properties for a data source, from Collibra's public documentation (best-effort scrape, not a stable API — see tool source for details). Use before edge_create_connection when the required parameters for a data source (e.g. Snowflake's Role/Warehouse/private key) aren't already known.", Handler: handler(), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/get_job_status/tool.go b/pkg/tools/get_job_status/tool.go index 2f7f4f2..3d36208 100644 --- a/pkg/tools/get_job_status/tool.go +++ b/pkg/tools/get_job_status/tool.go @@ -1,5 +1,7 @@ -// Package get_job_status implements the get_job_status MCP tool: polls the status of -// an Edge job, e.g. one started by test_connection or start_ingestion. +// Package get_job_status implements the get_job_status MCP tool: +// polls the status of a DGC job, e.g. the one returned by start_ingestion. +// This is distinct from edge_get_job_status, which polls Edge-site jobs (capability runs, +// connection tests) — the two job id spaces are not interchangeable. package get_job_status import ( @@ -13,20 +15,25 @@ import ( ) type Input struct { - JobID string `json:"jobId" jsonschema:"UUID of the job to check, as returned by test_connection or start_ingestion."` + JobID string `json:"jobId" jsonschema:"UUID of the DGC job to check, as returned by start_ingestion. Do not pass an Edge job id here (e.g. from test_connection) — use edge_get_job_status for those instead."` } type Output struct { - Status string `json:"status,omitempty" jsonschema:"The job's current status: UNKNOWN, SUBMITTED, DOWNLOADING, SCHEDULED, RUNNING, SUCCEEDED, FAILED, CANCELLED, CANCELLING, CAPABILITY_PROGRESS, CAPABILITY_SUCCEEDED, or CAPABILITY_FAILED. SUCCEEDED/FAILED/CANCELLED/CAPABILITY_SUCCEEDED/CAPABILITY_FAILED are terminal."` - Message string `json:"message,omitempty" jsonschema:"Status message from the job."` - LastUpdatedDateTime string `json:"lastUpdatedDateTime,omitempty" jsonschema:"When the status was last updated."` + Name string `json:"name,omitempty" jsonschema:"The job's name."` + Type string `json:"type,omitempty" jsonschema:"The job's type (e.g. 'DELTA_INGESTION')."` + State string `json:"state,omitempty" jsonschema:"The job's current state: WAITING, RUNNING, CANCELING, COMPLETED, CANCELED, or ERROR."` + Result string `json:"result,omitempty" jsonschema:"The job's result once finished: NOT_SET (not finished yet), SUCCESS, COMPLETED_WITH_ERROR, FAILURE, or ABORTED."` + Message string `json:"message,omitempty" jsonschema:"Status message from the job."` + ProgressPercentage int `json:"progressPercentage,omitempty" jsonschema:"Progress percentage, 0-100."` + StartDate string `json:"startDate,omitempty"` + EndDate string `json:"endDate,omitempty" jsonschema:"When the job finished, if it has."` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "get_job_status", - Title: "Get Edge Job Status", - Description: "Gets the current status of an Edge job, e.g. one started by test_connection or start_ingestion.", + Title: "Get Job Status", + Description: "Gets the current status of a DGC job, e.g. the ingestion job started by start_ingestion. For Edge-site jobs (test_connection, capability runs), use edge_get_job_status instead — the two job id spaces are separate and not interchangeable.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, @@ -39,11 +46,20 @@ func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { return Output{}, err } - status, err := clients.GetJobStatusLog(ctx, collibraClient, input.JobID) + job, err := clients.GetJob(ctx, collibraClient, input.JobID) if err != nil { return Output{}, err } - return Output{Status: status.Status, Message: status.Message, LastUpdatedDateTime: status.LastUpdatedDateTime}, nil + return Output{ + Name: job.Name, + Type: job.Type, + State: job.State, + Result: job.Result, + Message: job.Message, + ProgressPercentage: job.ProgressPercentage, + StartDate: job.StartDate, + EndDate: job.EndDate, + }, nil } } diff --git a/pkg/tools/get_job_status/tool_test.go b/pkg/tools/get_job_status/tool_test.go index 9d77209..e0a586e 100644 --- a/pkg/tools/get_job_status/tool_test.go +++ b/pkg/tools/get_job_status/tool_test.go @@ -5,24 +5,19 @@ import ( "net/http/httptest" "testing" - "github.com/collibra/chip/pkg/clients" tools "github.com/collibra/chip/pkg/tools/get_job_status" "github.com/collibra/chip/pkg/tools/testutil" "github.com/google/uuid" ) -func TestGetJobStatus(t *testing.T) { +func TestGetCatalogJobStatus(t *testing.T) { jobID, _ := uuid.NewUUID() handler := http.NewServeMux() - handler.Handle("GET /edge/api/rest/v2/jobs/"+jobID.String()+"/statusLog", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.JobStatusLog) { - return http.StatusOK, clients.JobStatusLog{ - JobID: jobID.String(), - Status: "CAPABILITY_SUCCEEDED", - Message: "done", - LastUpdatedDateTime: "2026-07-04T10:00:00.000Z", - } - })) + handler.HandleFunc("GET /rest/jobs/v1/jobs/"+jobID.String(), func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"` + jobID.String() + `","name":"Database Synchronisation of source","type":"DELTA_INGESTION","state":"COMPLETED","result":"SUCCESS","progressPercentage":100}`)) + }) server := httptest.NewServer(handler) defer server.Close() @@ -32,12 +27,12 @@ func TestGetJobStatus(t *testing.T) { if err != nil { t.Fatalf("expected no error, got: %v", err) } - if output.Status != "CAPABILITY_SUCCEEDED" { - t.Fatalf("unexpected status: %s", output.Status) + if output.State != "COMPLETED" || output.Result != "SUCCESS" { + t.Fatalf("unexpected output: %+v", output) } } -func TestGetJobStatus_InvalidJobID(t *testing.T) { +func TestGetCatalogJobStatus_InvalidJobID(t *testing.T) { client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{JobID: "not-a-uuid"}) if err == nil { diff --git a/pkg/tools/jobs_find/tool.go b/pkg/tools/jobs_find/tool.go index d6e3586..8d0ee57 100644 --- a/pkg/tools/jobs_find/tool.go +++ b/pkg/tools/jobs_find/tool.go @@ -25,9 +25,9 @@ type Input struct { type Output struct { Jobs []clients.Job `json:"jobs,omitempty" jsonschema:"list of matching jobs"` - Count int `json:"count" jsonschema:"number of jobs returned"` - NextCursor string `json:"nextCursor,omitempty" jsonschema:"cursor for the next page of results"` - Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` + Count int `json:"count" jsonschema:"number of jobs returned"` + NextCursor string `json:"nextCursor,omitempty" jsonschema:"cursor for the next page of results"` + Error string `json:"error,omitempty" jsonschema:"error message if retrieval failed"` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { diff --git a/pkg/tools/list_integrations/tool.go b/pkg/tools/list_integrations/tool.go index 2a1c17b..7533e87 100644 --- a/pkg/tools/list_integrations/tool.go +++ b/pkg/tools/list_integrations/tool.go @@ -19,18 +19,18 @@ type Input struct { } type Integration struct { - IngestibleId string `json:"ingestibleId" jsonschema:"UUID of the integration instance, use this for all subsequent calls"` - Name string `json:"name" jsonschema:"capability name"` - TypeId string `json:"typeId" jsonschema:"capability type, e.g. databricks-edge-capability"` - EdgeSiteId string `json:"edgeSiteId" jsonschema:"Edge site this capability runs on"` - HasSchedule bool `json:"hasSchedule" jsonschema:"whether a sync schedule is configured"` - CronExpression string `json:"cronExpression,omitempty" jsonschema:"cron expression for the schedule"` - CronTimeZone string `json:"cronTimeZone,omitempty" jsonschema:"timezone for the cron schedule"` - LastRun string `json:"lastRun,omitempty" jsonschema:"relative time of last sync run, e.g. 6h ago"` - LastRunAt string `json:"lastRunAt,omitempty" jsonschema:"ISO 8601 timestamp of last sync run, use this for precise time comparisons e.g. ran in last 24h"` - LastRunState string `json:"lastRunState,omitempty" jsonschema:"state of last job: COMPLETED, FAILED, RUNNING, etc."` - LastRunResult string `json:"lastRunResult,omitempty" jsonschema:"result of last job: SUCCESS, FAILURE, COMPLETED_WITH_ERROR, etc."` - NextRun string `json:"nextRun,omitempty" jsonschema:"relative time of next scheduled run, e.g. in 18h"` + IngestibleId string `json:"ingestibleId" jsonschema:"UUID of the integration instance, use this for all subsequent calls"` + Name string `json:"name" jsonschema:"capability name"` + TypeId string `json:"typeId" jsonschema:"capability type, e.g. databricks-edge-capability"` + EdgeSiteId string `json:"edgeSiteId" jsonschema:"Edge site this capability runs on"` + HasSchedule bool `json:"hasSchedule" jsonschema:"whether a sync schedule is configured"` + CronExpression string `json:"cronExpression,omitempty" jsonschema:"cron expression for the schedule"` + CronTimeZone string `json:"cronTimeZone,omitempty" jsonschema:"timezone for the cron schedule"` + LastRun string `json:"lastRun,omitempty" jsonschema:"relative time of last sync run, e.g. 6h ago"` + LastRunAt string `json:"lastRunAt,omitempty" jsonschema:"ISO 8601 timestamp of last sync run, use this for precise time comparisons e.g. ran in last 24h"` + LastRunState string `json:"lastRunState,omitempty" jsonschema:"state of last job: COMPLETED, FAILED, RUNNING, etc."` + LastRunResult string `json:"lastRunResult,omitempty" jsonschema:"result of last job: SUCCESS, FAILURE, COMPLETED_WITH_ERROR, etc."` + NextRun string `json:"nextRun,omitempty" jsonschema:"relative time of next scheduled run, e.g. in 18h"` } type Output struct { @@ -44,7 +44,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { Name: "list_integrations", Description: `Lists all integration capabilities with their sync schedule and last/next run times. Use this as the entry point for any integration question: "show me all integrations", "show Databricks integrations", "what's scheduled for the Finance integration?". -Optionally filter by platform substring (e.g. "databricks", "dataplex") and/or name. Returns ingestibleId for each — pass it to catalog_generic_get_schedule, catalog_generic_start_job, or catalog_generic_cancel_job. +Optionally filter by platform substring (e.g. "databricks", "dataplex") and/or name. Returns ingestibleId for each — pass it to catalog_etl_get_schedule, catalog_etl_start_job, or catalog_etl_cancel_job. For time-based queries like "which integrations ran in the last 24h", use jobs_find sorted by startDate to get recent jobs, then cross-reference with integration names from this tool.`, Handler: handler(collibraClient), Permissions: []string{}, diff --git a/pkg/tools/list_integrations/tool_test.go b/pkg/tools/list_integrations/tool_test.go index 40ebda5..71895d6 100644 --- a/pkg/tools/list_integrations/tool_test.go +++ b/pkg/tools/list_integrations/tool_test.go @@ -85,9 +85,9 @@ func TestListIntegrationsScheduleEnrichment(t *testing.T) { })) handler.Handle("/rest/catalog/1.0/genericIntegration/00000000-0000-0000-0000-000000000001/schedule", testutil.JsonHandlerOut(func(r *http.Request) (int, clients.GenericSchedule) { return http.StatusOK, clients.GenericSchedule{ - CronExpression: "0 2 * * *", - CronTimeZone: "UTC", - LastRunTimeStamp: 1000, + CronExpression: "0 2 * * *", + CronTimeZone: "UTC", + LastRunTimeStamp: 1000, NextRunDateLongValue: 9999999999000, } })) diff --git a/pkg/tools/register.go b/pkg/tools/register.go index 3512069..015ef09 100644 --- a/pkg/tools/register.go +++ b/pkg/tools/register.go @@ -7,47 +7,49 @@ import ( "github.com/collibra/chip/pkg/chip" "github.com/collibra/chip/pkg/skills" "github.com/collibra/chip/pkg/tools/add_data_classification_match" - "github.com/collibra/chip/pkg/tools/catalog_generic_add_schedule" - "github.com/collibra/chip/pkg/tools/catalog_generic_cancel_job" - "github.com/collibra/chip/pkg/tools/catalog_generic_delete_config" - "github.com/collibra/chip/pkg/tools/catalog_generic_delete_schedule" - "github.com/collibra/chip/pkg/tools/catalog_generic_get_all_schedules" - "github.com/collibra/chip/pkg/tools/catalog_generic_get_config" - "github.com/collibra/chip/pkg/tools/catalog_generic_get_schedule" - "github.com/collibra/chip/pkg/tools/catalog_generic_get_schema" - "github.com/collibra/chip/pkg/tools/catalog_generic_save_config" - "github.com/collibra/chip/pkg/tools/catalog_generic_start_job" - "github.com/collibra/chip/pkg/tools/catalog_generic_update_schedule" + "github.com/collibra/chip/pkg/tools/catalog_etl_add_schedule" + "github.com/collibra/chip/pkg/tools/catalog_etl_cancel_job" + "github.com/collibra/chip/pkg/tools/catalog_etl_delete_generic_config" + "github.com/collibra/chip/pkg/tools/catalog_etl_delete_schedule" + "github.com/collibra/chip/pkg/tools/catalog_etl_get_all_schedules" + "github.com/collibra/chip/pkg/tools/catalog_etl_get_config" + "github.com/collibra/chip/pkg/tools/catalog_etl_get_schedule" + "github.com/collibra/chip/pkg/tools/catalog_etl_get_schema" + "github.com/collibra/chip/pkg/tools/catalog_etl_save_generic_config" + "github.com/collibra/chip/pkg/tools/catalog_etl_start_job" + "github.com/collibra/chip/pkg/tools/catalog_etl_update_schedule" "github.com/collibra/chip/pkg/tools/configure_database" "github.com/collibra/chip/pkg/tools/create_asset" - "github.com/collibra/chip/pkg/tools/create_capability" "github.com/collibra/chip/pkg/tools/create_community" - "github.com/collibra/chip/pkg/tools/create_connection" "github.com/collibra/chip/pkg/tools/create_domain" "github.com/collibra/chip/pkg/tools/discover_business_glossary" "github.com/collibra/chip/pkg/tools/discover_data_assets" "github.com/collibra/chip/pkg/tools/edge_cancel_job" + "github.com/collibra/chip/pkg/tools/edge_create_capability" + "github.com/collibra/chip/pkg/tools/edge_create_connection" "github.com/collibra/chip/pkg/tools/edge_delete_capability" "github.com/collibra/chip/pkg/tools/edge_delete_connection" "github.com/collibra/chip/pkg/tools/edge_find_capabilities" + "github.com/collibra/chip/pkg/tools/edge_find_connections" "github.com/collibra/chip/pkg/tools/edge_get_capability" "github.com/collibra/chip/pkg/tools/edge_get_connection" + "github.com/collibra/chip/pkg/tools/edge_get_job_status" "github.com/collibra/chip/pkg/tools/edge_get_job_status_history" "github.com/collibra/chip/pkg/tools/edge_list_capabilities" + "github.com/collibra/chip/pkg/tools/edge_list_capability_types" "github.com/collibra/chip/pkg/tools/edge_list_connections" + "github.com/collibra/chip/pkg/tools/edge_list_sites" "github.com/collibra/chip/pkg/tools/edge_run_capability" "github.com/collibra/chip/pkg/tools/edit_asset" - "github.com/collibra/chip/pkg/tools/find_connections" "github.com/collibra/chip/pkg/tools/find_domain_types" "github.com/collibra/chip/pkg/tools/find_users" "github.com/collibra/chip/pkg/tools/get_asset_details" "github.com/collibra/chip/pkg/tools/get_business_term_data" - "github.com/collibra/chip/pkg/tools/get_catalog_job_status" + "github.com/collibra/chip/pkg/tools/get_job_status" "github.com/collibra/chip/pkg/tools/get_column_semantics" "github.com/collibra/chip/pkg/tools/get_context_specification" "github.com/collibra/chip/pkg/tools/get_data_source_setup_guide" "github.com/collibra/chip/pkg/tools/get_debug_mcp_init_request" - "github.com/collibra/chip/pkg/tools/get_job_status" "github.com/collibra/chip/pkg/tools/get_lineage_downstream" "github.com/collibra/chip/pkg/tools/get_lineage_entity" "github.com/collibra/chip/pkg/tools/get_lineage_transformation" @@ -57,10 +59,8 @@ import ( "github.com/collibra/chip/pkg/tools/init_data_contract" "github.com/collibra/chip/pkg/tools/jobs_find" "github.com/collibra/chip/pkg/tools/list_asset_types" - "github.com/collibra/chip/pkg/tools/list_capability_types" "github.com/collibra/chip/pkg/tools/list_context_specifications" "github.com/collibra/chip/pkg/tools/list_data_contracts" - "github.com/collibra/chip/pkg/tools/list_edge_sites" "github.com/collibra/chip/pkg/tools/list_integrations" "github.com/collibra/chip/pkg/tools/prepare_create_asset" "github.com/collibra/chip/pkg/tools/pull_data_contract_manifest" @@ -121,9 +121,9 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv } toolRegister(server, toolConfig, find_users.NewTool(client)) toolRegister(server, toolConfig, find_domain_types.NewTool(client)) - toolRegister(server, toolConfig, find_connections.NewTool(client)) - toolRegister(server, toolConfig, list_edge_sites.NewTool(client)) - toolRegister(server, toolConfig, list_capability_types.NewTool(client)) + toolRegister(server, toolConfig, edge_find_connections.NewTool(client)) + toolRegister(server, toolConfig, edge_list_sites.NewTool(client)) + toolRegister(server, toolConfig, edge_list_capability_types.NewTool(client)) toolRegister(server, toolConfig, create_community.NewTool(client)) toolRegister(server, toolConfig, create_domain.NewTool(client)) if toolConfig.AllowLocalFileUpload { @@ -132,17 +132,17 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, upload_file.NewTool(client)) } toolRegister(server, toolConfig, get_data_source_setup_guide.NewTool(client)) - toolRegister(server, toolConfig, create_connection.NewTool(client)) - toolRegister(server, toolConfig, create_capability.NewTool(client)) + toolRegister(server, toolConfig, edge_create_connection.NewTool(client)) + toolRegister(server, toolConfig, edge_create_capability.NewTool(client)) toolRegister(server, toolConfig, test_connection.NewTool(client)) + toolRegister(server, toolConfig, edge_get_job_status.NewTool(client)) toolRegister(server, toolConfig, get_job_status.NewTool(client)) - toolRegister(server, toolConfig, get_catalog_job_status.NewTool(client)) toolRegister(server, toolConfig, configure_database.NewTool(client)) toolRegister(server, toolConfig, start_ingestion.NewTool(client)) // Edge Management tools — read/delete/run operations with no counterpart among the - // create/find/test/get-status lifecycle tools above (create_connection and - // create_capability already upsert, so no edge update tools are needed here). + // create/find/test/get-status lifecycle tools above (edge_create_connection and + // edge_create_capability already upsert, so no edge update tools are needed here). toolRegister(server, toolConfig, edge_list_capabilities.NewTool(client)) toolRegister(server, toolConfig, edge_find_capabilities.NewTool(client)) toolRegister(server, toolConfig, edge_get_capability.NewTool(client)) @@ -154,17 +154,17 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, edge_cancel_job.NewTool(client)) toolRegister(server, toolConfig, edge_get_job_status_history.NewTool(client)) // Catalog Generic Integration tools - toolRegister(server, toolConfig, catalog_generic_get_config.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_save_config.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_delete_config.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_get_schema.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_get_schedule.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_add_schedule.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_update_schedule.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_delete_schedule.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_get_all_schedules.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_cancel_job.NewTool(client)) - toolRegister(server, toolConfig, catalog_generic_start_job.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_get_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_save_generic_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_delete_generic_config.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_get_schema.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_get_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_add_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_update_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_delete_schedule.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_get_all_schedules.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_cancel_job.NewTool(client)) + toolRegister(server, toolConfig, catalog_etl_start_job.NewTool(client)) // Jobs tools toolRegister(server, toolConfig, jobs_find.NewTool(client)) // Integration lifecycle tools diff --git a/pkg/tools/start_ingestion/tool.go b/pkg/tools/start_ingestion/tool.go index 0b99178..aa2b263 100644 --- a/pkg/tools/start_ingestion/tool.go +++ b/pkg/tools/start_ingestion/tool.go @@ -19,16 +19,16 @@ type Input struct { } type Output struct { - Job *clients.CatalogJob `json:"job,omitempty" jsonschema:"The started synchronization job. Poll its id with get_catalog_job_status (NOT get_job_status, which is for Edge-site jobs) to see when it completes."` - Success bool `json:"success" jsonschema:"Whether the ingestion job was started."` - Error string `json:"error,omitempty" jsonschema:"Error message if starting the job failed."` + Job *clients.CatalogJob `json:"job,omitempty" jsonschema:"The started synchronization job. Poll its id with get_job_status (NOT edge_get_job_status, which is for Edge-site jobs) to see when it completes."` + Success bool `json:"success" jsonschema:"Whether the ingestion job was started."` + Error string `json:"error,omitempty" jsonschema:"Error message if starting the job failed."` } func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "start_ingestion", Title: "Start Database Ingestion", - Description: "Triggers the jdbc-ingestion capability run for a registered Database asset, synchronizing its configured schemas/tables into the catalog. The database must already be registered and configured via configure_database. Poll the returned job's id with get_catalog_job_status to confirm it actually completed — a 202/success response here only means the job was accepted, not that ingestion finished.", + Description: "Triggers the jdbc-ingestion capability run for a registered Database asset, synchronizing its configured schemas/tables into the catalog. The database must already be registered and configured via configure_database. Poll the returned job's id with get_job_status to confirm it actually completed — a 202/success response here only means the job was accepted, not that ingestion finished.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/test_connection/tool.go b/pkg/tools/test_connection/tool.go index bbf70bf..94a9873 100644 --- a/pkg/tools/test_connection/tool.go +++ b/pkg/tools/test_connection/tool.go @@ -14,13 +14,13 @@ import ( ) type Input struct { - ConnectionID string `json:"connectionId" jsonschema:"UUID of the connection to test (created via create_connection)."` - TimeoutSec int `json:"timeoutSec,omitempty" jsonschema:"Optional. If provided and greater than 0, waits synchronously for the test to finish (up to this many seconds) and returns the final result. If omitted, returns immediately with a jobId while the test runs in the background — poll it with get_job_status."` + ConnectionID string `json:"connectionId" jsonschema:"UUID of the connection to test (created via edge_create_connection)."` + TimeoutSec int `json:"timeoutSec,omitempty" jsonschema:"Optional. If provided and greater than 0, waits synchronously for the test to finish (up to this many seconds) and returns the final result. If omitted, returns immediately with a jobId while the test runs in the background — poll it with edge_get_job_status."` } type Output struct { - JobID string `json:"jobId,omitempty" jsonschema:"The id of the connection-test job. Use with get_job_status to poll for the result if timeoutSec was not provided."` - Success bool `json:"success" jsonschema:"Whether the test connection request itself succeeded. If timeoutSec was omitted, this reflects that the test job was submitted, not the connection test's outcome — check get_job_status for that."` + JobID string `json:"jobId,omitempty" jsonschema:"The id of the connection-test job. Use with edge_get_job_status to poll for the result if timeoutSec was not provided."` + Success bool `json:"success" jsonschema:"Whether the test connection request itself succeeded. If timeoutSec was omitted, this reflects that the test job was submitted, not the connection test's outcome — check edge_get_job_status for that."` Message string `json:"message,omitempty" jsonschema:"Status message from the test."` Error string `json:"error,omitempty" jsonschema:"Error message if the test request itself failed."` } @@ -29,7 +29,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "test_connection", Title: "Test Edge Connection", - Description: "Tests whether an Edge connection can reach its data source. Without timeoutSec, returns immediately with a jobId to poll via get_job_status; with timeoutSec, waits up to that many seconds and returns the final result.", + Description: "Tests whether an Edge connection can reach its data source. Without timeoutSec, returns immediately with a jobId to poll via edge_get_job_status; with timeoutSec, waits up to that many seconds and returns the final result.", Handler: handler(collibraClient), Permissions: []string{"dgc.edge-integration-capability-manage"}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/upload_file/tool.go b/pkg/tools/upload_file/tool.go index 50c711d..7295b29 100644 --- a/pkg/tools/upload_file/tool.go +++ b/pkg/tools/upload_file/tool.go @@ -49,9 +49,9 @@ // catalog-backed driver API (POST /rest/edge/v1/drivers/jars), which has no UI in // front of it at all, and the frontend's only file-upload path is the legacy one // embedded inside the connection-creation form's own FILE-type field. Once the user -// confirms the connection is saved, use find_connections (by name) to pick up its id -// and proceed with create_capability/configure_database/start_ingestion against it — -// skip create_connection entirely for that connection, since it already exists. +// confirms the connection is saved, use edge_find_connections (by name) to pick up its id +// and proceed with edge_create_capability/configure_database/start_ingestion against it — +// skip edge_create_connection entirely for that connection, since it already exists. package upload_file import ( @@ -70,7 +70,7 @@ import ( // case for chip-service) — content must be supplied inline as base64. type Input struct { Filename string `json:"filename" jsonschema:"The filename to upload as (e.g. 'client-cert.pem', 'snowflake-jdbc-3.23.2.jar'). Determines the file extension DGC uses to infer the artifact type."` - ContentBase64 string `json:"contentBase64" jsonschema:"The file's content, base64-encoded. For a JDBC driver, this must come from a file the user downloaded from Collibra Marketplace (see get_data_source_setup_guide's marketplaceUrl) — never fetch a driver from an arbitrary URL like Maven Central; ask the user for the file instead. Large files (tens of MB, as most JDBC drivers are) will not fit here — if the content would be too large to include, ask the user to create the connection manually via the Collibra Edge UI instead and pick it up afterward with find_connections."` + ContentBase64 string `json:"contentBase64" jsonschema:"The file's content, base64-encoded. For a JDBC driver, this must come from a file the user downloaded from Collibra Marketplace (see get_data_source_setup_guide's marketplaceUrl) — never fetch a driver from an arbitrary URL like Maven Central; ask the user for the file instead. Large files (tens of MB, as most JDBC drivers are) will not fit here — if the content would be too large to include, ask the user to create the connection manually via the Collibra Edge UI instead and pick it up afterward with edge_find_connections."` } // InputWithFilePath is used when AllowLocalFileUpload is true (only ever set by the @@ -94,7 +94,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "upload_file", Title: "Upload File to Edge Site", - Description: "Uploads a file (JDBC driver, TLS certificate, private key, keytab, etc.) to an edge site and returns an artifact URI usable as the value of any FILE-type connection or capability parameter. Takes file content directly (base64) — does not fetch from a URL. Only suitable for small files (contentBase64 has to fit in a tool call); for a large file (e.g. most JDBC drivers), guide the user to create the connection manually via the Collibra Edge UI and pick it up with find_connections. For JDBC drivers, only use content downloaded from Collibra Marketplace (verified/supported), never an arbitrary host; get the Marketplace listing URL from get_data_source_setup_guide.", + Description: "Uploads a file (JDBC driver, TLS certificate, private key, keytab, etc.) to an edge site and returns an artifact URI usable as the value of any FILE-type connection or capability parameter. Takes file content directly (base64) — does not fetch from a URL. Only suitable for small files (contentBase64 has to fit in a tool call); for a large file (e.g. most JDBC drivers), guide the user to create the connection manually via the Collibra Edge UI and pick it up with edge_find_connections. For JDBC drivers, only use content downloaded from Collibra Marketplace (verified/supported), never an arbitrary host; get the Marketplace listing URL from get_data_source_setup_guide.", Handler: func(ctx context.Context, input Input) (Output, error) { return upload(ctx, collibraClient, input.Filename, input.ContentBase64, "") }, From c7f1837e6224cd3a6da5bcae48632424c315bd6e Mon Sep 17 00:00:00 2001 From: Dary Schaeken Date: Wed, 8 Jul 2026 10:32:15 +0200 Subject: [PATCH 09/10] refactor: split configure_database into register_database + configure_database_schemas configure_database registered the Database asset and configured schema sync rules in one call, calling POST /databases before resolving schema ambiguity. A schema-ambiguity error left the Database asset already created, so retrying with schemaNames set failed with databaseConnectionAlreadyUsed instead of resuming. Splits into register_database (discovers the database, registers it) and configure_database_schemas (discovers schemas, sets sync rules, keyed on register_database's returned databaseConnectionId). Neither has a side effect until its own ambiguity is resolved, so an ambiguity error is now a true no-op, and schema sync rules can be reconfigured independently later without re-registering the database. Also trims internal policy detail (specific competitor names) from the jdbc-ingestion skill's user-facing driver-sourcing guidance, keeping the operational instruction without the implementation reasoning. --- README.md | 3 +- cmd/chip/experimental.go | 2 +- .../files/collibra/etl-integration/SKILL.md | 5 +- .../files/collibra/jdbc-ingestion/SKILL.md | 102 +++---- pkg/tools/configure_database/tool.go | 266 ------------------ pkg/tools/configure_database_schemas/tool.go | 186 ++++++++++++ .../tool_test.go | 142 ++-------- pkg/tools/create_community/tool.go | 4 +- pkg/tools/create_domain/tool.go | 4 +- pkg/tools/find_users/tool.go | 4 +- pkg/tools/register.go | 8 +- pkg/tools/register_database/tool.go | 145 ++++++++++ pkg/tools/register_database/tool_test.go | 188 +++++++++++++ pkg/tools/start_ingestion/tool.go | 6 +- pkg/tools/upload_file/tool.go | 7 +- 15 files changed, 628 insertions(+), 444 deletions(-) delete mode 100644 pkg/tools/configure_database/tool.go create mode 100644 pkg/tools/configure_database_schemas/tool.go rename pkg/tools/{configure_database => configure_database_schemas}/tool_test.go (52%) create mode 100644 pkg/tools/register_database/tool.go create mode 100644 pkg/tools/register_database/tool_test.go diff --git a/README.md b/README.md index dcd6592..d75dc36 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, ### Write Tools - [`add_data_classification_match`](pkg/tools/add_data_classification_match/) - Associate a data class with an asset. **Requires:** `dgc.classify`, `dgc.catalog` -- [`configure_database`](pkg/tools/configure_database/) - Register a database and configure its schema synchronization rules for jdbc-ingestion +- [`configure_database_schemas`](pkg/tools/configure_database_schemas/) - Discover and configure schema/table synchronization rules for a database registered via register_database, for jdbc-ingestion - [`create_asset`](pkg/tools/create_asset/) - Create a new asset of any type. Resolves `assetType` (UUID, publicId, or display name), `domain` (UUID or name), `status` (UUID or name), and attributes (by name or typeId) server-side; converts Markdown to HTML for `RICH_TEXT` attributes; gates on duplicate-name (default `allowDuplicate: false`) - [`edge_create_capability`](pkg/tools/edge_create_capability/) - Create or update an Edge capability (e.g. jdbc-ingestion). **Requires:** `dgc.edge-integration-capability-manage` - [`create_community`](pkg/tools/create_community/) - Create a DGC community @@ -58,6 +58,7 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - `remove_responsibility` - unassign a user or group from a resource role (only directly-assigned responsibilities, not inherited ones) - [`init_data_contract`](pkg/tools/init_data_contract/) - Initialize a new data contract asset governing a Data Product Port, with an optional initial manifest. **Requires:** `dgc.data-contract` - [`push_data_contract_manifest`](pkg/tools/push_data_contract_manifest/) - Upload manifest for a data contract. **Requires:** `dgc.data-contract` +- [`register_database`](pkg/tools/register_database/) - Discover a database through an Edge connection and register it as a Database asset, for jdbc-ingestion - [`remove_data_classification_match`](pkg/tools/remove_data_classification_match/) - Remove a classification match. **Requires:** `dgc.classify`, `dgc.catalog`, `dgc.data-classes-edit` - [`start_ingestion`](pkg/tools/start_ingestion/) - Trigger a jdbc-ingestion capability run for a registered database - [`upload_file`](pkg/tools/upload_file/) - Upload a file (JDBC driver, TLS certificate, private key, keytab, …) to an Edge site. **Requires:** `dgc.edge-integration-capability-manage` diff --git a/cmd/chip/experimental.go b/cmd/chip/experimental.go index faf0a22..874f06f 100644 --- a/cmd/chip/experimental.go +++ b/cmd/chip/experimental.go @@ -17,7 +17,7 @@ import ( // the help text, example YAML, and validation warning all read from this // map so no other code needs to change. var knownExperimentalFeatures = map[string]string{ - skills.FeatureName: "Embedded skill catalog served via list_collibra_skills and load_collibra_skill.", + skills.FeatureName: "Embedded skill catalog served via list_collibra_skills and load_collibra_skill.", tools.ContextSpecificationsFeature: "Context specification tools: list_context_specifications, get_context_specification, and contextSpecificationId parameter on get_asset_details.", } diff --git a/pkg/skills/files/collibra/etl-integration/SKILL.md b/pkg/skills/files/collibra/etl-integration/SKILL.md index 231ab90..71d17b3 100644 --- a/pkg/skills/files/collibra/etl-integration/SKILL.md +++ b/pkg/skills/files/collibra/etl-integration/SKILL.md @@ -136,8 +136,9 @@ replace,** the live schema from `catalog_etl_get_schema`. ## Distinct from jdbc-ingestion `collibra/jdbc-ingestion` is a specialized flow: it registers a Database asset via -`configure_database` and runs via `start_ingestion` (DGC's `synchronizeMetadata`), **not** -`catalog_etl_start_job`. Use that skill for JDBC database ingestion; use **this** skill for +`register_database`, configures schemas via `configure_database_schemas`, and runs via +`start_ingestion` (DGC's `synchronizeMetadata`), **not** `catalog_etl_start_job`. Use +that skill for JDBC database ingestion; use **this** skill for the generic/ETL sync capabilities (Dataplex, Databricks Unity Catalog, Purview, Sigma, ThoughtSpot, …). diff --git a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md index 78f4b9d..ca5e552 100644 --- a/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md +++ b/pkg/skills/files/collibra/jdbc-ingestion/SKILL.md @@ -35,12 +35,12 @@ which ingestion type it should be** — don't assume JDBC just because this skil string format, the Marketplace URL for the driver jar, and per-auth-method properties. Treat a failed/empty lookup as "ask the user for the connection properties directly," not as a bug to retry. -3. **`upload_file`** — upload the JDBC driver jar. **Driver files must come from - Collibra Marketplace only** (the URL from step 2) — never fetch one from Maven - Central or another host, and never invent one. If the file is too large for - `contentBase64` (most JDBC drivers are tens of MB) and this chip instance doesn't - have `filePath` enabled, stop and guide the user to create the connection manually - in the Edge UI instead — see "Large driver files" below. +3. **`upload_file`** — upload the JDBC driver jar. **Only use a file the user actually + provides, sourced from the Collibra Marketplace listing from step 2** — never fetch + a driver yourself from anywhere else, and never invent one. If the file is too large + for `contentBase64` (most JDBC drivers are tens of MB) and this chip instance + doesn't have `filePath` enabled, stop and guide the user to create the connection + manually in the Edge UI instead — see "Large driver files" below. 4. **`edge_create_connection`** — `typeId: "Generic"` (not a vendor-specific connection type — jdbc-ingestion requires the Generic JDBC connection type). Fixed manifest parameters (`driver-class`, `connection-string`, `driver-jar` as the `upload_file` @@ -55,17 +55,23 @@ which ingestion type it should be** — don't assume JDBC just because this skil **`find_domain_types`** (look up e.g. "Physical Data Dictionary") → **`create_domain`**. **`find_users`** to resolve owner names to UUIDs for the next step. -8. **`configure_database`** — one call does the full refresh-connection → - register-database → refresh-schema → set-sync-rules flow. Needs `communityId`, - `parentSystemId` (an existing System asset), and `ownerIds` (from step 7). **Never - default to "sync everything" — always confirm which database, which schemas, and - which tables with the user first.** See "Confirm scope before configuring the - database" below; the tool itself refuses to guess when there's more than one - database or schema. If it returns a retryable-looking error, it usually means the - underlying async refresh hasn't completed yet (cold-start data source connections - can take 30–60s) — just call it again; it's idempotent. -9. **`start_ingestion`** — triggers the actual sync using the database id from step 8. -10. **`get_job_status`** — poll the job id `start_ingestion` returned. A +8. **`register_database`** — discovers the database through the Edge connection and + registers it as a Database asset. Needs `communityId`, `parentSystemId` (an existing + System asset), and `ownerIds` (from step 7). **Never default to registering + whichever database is discovered first if more than one exists — always confirm + which one with the user.** The tool refuses to guess when there's more than one + database (see "Confirm scope" below). If it returns a retryable-looking error, it + usually means the underlying async refresh hasn't completed yet (cold-start data + source connections can take 30–60s) — just call it again; it's idempotent. +9. **`configure_database_schemas`** — takes the `databaseConnectionId` from step 8's output, + discovers its schemas, and sets which schemas/tables get synchronized. **Never + default to "sync everything" — always confirm which schemas and which tables with + the user first.** The tool refuses to guess when there's more than one schema. This + is a separate call from step 8 specifically so that a schema-ambiguity error here + never risks colliding with the database step 8 already registered — safe to call + again later to change what's synced. +10. **`start_ingestion`** — triggers the actual sync using the database id from step 8. +11. **`get_job_status`** — poll the job id `start_ingestion` returned. A 202/success from `start_ingestion` only means the job was accepted, not that ingestion finished — always poll to confirm completion. @@ -88,30 +94,32 @@ catalog database-registration bookkeeping (linking the job back to the Database updating sync state) that a raw capability run skips entirely — a direct run would appear to succeed on the Edge side while leaving the catalog side out of sync. -## Confirm scope before configuring the database - -`configure_database` will not silently register the wrong database or sync every table -in every schema — it's built to refuse ambiguity rather than guess: - -- **`databaseName`** is required if the data source exposes more than one - database/catalog through the connection. The tool errors and names the discovered - candidates if you omit it while more than one exists. -- **`schemaNames`** is required if the database has more than one schema. Same - behavior: omit it with multiple schemas discovered and the tool errors, listing them. - Pass the literal `["*"]` to configure every schema — only do this after the user has - actually said they want everything, not as a shortcut when you don't know yet. -- **`include`** has no default. You must always pass a table pattern (`"*"` for all - tables, or a specific comma-separated pattern) — there is no silent "sync - everything" fallback. - -The practical flow: call `configure_database` once with just the required identifiers -and let it fail on ambiguity to discover the candidate databases/schemas, **or** if you -already know multiple exist, ask the user up front which database, which schemas, and -which tables (or "all") they want before calling at all. Either way, do not pass `"*"` -for `schemaNames` or `include` on your own initiative — confirm with the user first. -The one exception: if discovery finds exactly one database and one schema, the tool -auto-selects them — no ambiguity to resolve, so no need to ask about *which* database -or schema, though you should still confirm the table pattern (`include`/`exclude`). +## Confirm scope before registering and configuring the database + +`register_database` and `configure_database_schemas` will not silently register the wrong +database or sync every table in every schema — both refuse ambiguity rather than guess, +and neither has a side effect until its own ambiguity is resolved: + +- **`register_database`'s `databaseName`** is required if the data source exposes more + than one database/catalog through the connection. The tool errors and names the + discovered candidates if you omit it while more than one exists — no Database asset + is registered until this is resolved. +- **`configure_database_schemas`'s `schemaNames`** is required if the database has more than + one schema. Same behavior: omit it with multiple schemas discovered and the tool + errors, listing them. Pass the literal `["*"]` to configure every schema — only do + this after the user has actually said they want everything, not as a shortcut when + you don't know yet. +- **`configure_database_schemas`'s `include`** has no default. You must always pass a table + pattern (`"*"` for all tables, or a specific comma-separated pattern) — there is no + silent "sync everything" fallback. + +The practical flow for each tool: call it once with just the required identifiers and +let it fail on ambiguity to discover the candidates, **or** if you already know +multiple exist, ask the user up front before calling at all. Either way, do not pass +`"*"` for `schemaNames` or `include` on your own initiative — confirm with the user +first. The one exception: if discovery finds exactly one database (or one schema), the +respective tool auto-selects it — no ambiguity to resolve, so no need to ask about +*which* one, though you should still confirm the table pattern (`include`/`exclude`). ## Picking up an existing connection instead of duplicating one @@ -135,14 +143,14 @@ need to pick the file and click through. Once they confirm it's saved, use ## Hard rules -1. **Never source a JDBC driver from anywhere but Collibra Marketplace.** Not Maven - Central, not a GitHub release, not a URL the user pastes that isn't the Marketplace - listing. If in doubt, ask the user to download the jar from the URL - `get_data_source_setup_guide` returns and hand you the file. +1. **Never source a JDBC driver from anywhere but Collibra Marketplace.** If in doubt, + ask the user to download the jar from the URL `get_data_source_setup_guide` returns + and hand you the file — don't fetch or substitute one yourself. 2. **`edge_create_connection`'s `typeId` for jdbc-ingestion is `"Generic"`,** never a vendor-specific connection type — jdbc-ingestion cannot use those. 3. **Always poll after `start_ingestion`.** A successful call only means the job was accepted. 4. **Never pick a database, schema set, or table scope for the user.** When more than - one is possible, ask; when `configure_database` reports ambiguity, relay the - candidates and ask rather than picking one yourself or retrying with `"*"`. + one is possible, ask; when `register_database` or `configure_database_schemas` reports + ambiguity, relay the candidates and ask rather than picking one yourself or + retrying with `"*"`. diff --git a/pkg/tools/configure_database/tool.go b/pkg/tools/configure_database/tool.go deleted file mode 100644 index 94d8306..0000000 --- a/pkg/tools/configure_database/tool.go +++ /dev/null @@ -1,266 +0,0 @@ -// Package configure_database implements the configure_database MCP tool: registers a -// database discovered through an Edge connection as a Database asset and configures -// which schemas/tables get synchronized, via the public catalogDatabase API. -// -// This wraps a 4-step flow (refresh database connections, register the database, -// refresh schema connections, set synchronization rules) as a single tool call. The -// refresh steps are asynchronous on the server, so this tool polls for their result -// with a bounded number of attempts; if a stage hasn't completed within that budget, -// it returns an error asking the agent to retry the tool call (the refresh calls are -// idempotent to re-issue). -// -// Ambiguity guards: databaseName is required when more than one database/catalog is -// discovered, schemaNames is required when more than one schema is discovered, and -// include has no default — all three force an explicit choice instead of silently -// registering the wrong database or syncing every schema/table. schemaNames/include -// still accept an explicit "*" to opt into "everything", but that must be a deliberate -// choice (confirmed with the user), never this tool's default behavior. -package configure_database - -import ( - "context" - "fmt" - "net/http" - "strings" - "time" - - "github.com/collibra/chip/pkg/chip" - "github.com/collibra/chip/pkg/clients" - "github.com/collibra/chip/pkg/tools/validation" - "github.com/modelcontextprotocol/go-sdk/mcp" -) - -// Sleep is used for polling backoff. Overridable in tests to avoid slow test runs. -var Sleep = time.Sleep - -const ( - // pollAttempts * pollInterval bounds the wait for each async refresh. A - // first-time connection to a data source can take 30-60s to establish, so - // this budget needs enough headroom to cover that cold-start case. - pollAttempts = 14 - pollInterval = 5 * time.Second -) - -type Input struct { - EdgeConnectionID string `json:"edgeConnectionId" jsonschema:"UUID of the Edge connection (created via edge_create_connection) to discover a database on."` - DatabaseName string `json:"databaseName,omitempty" jsonschema:"Optional. Exact name of the database (catalog) to register, as it appears at the data source. Required if the data source exposes more than one database/catalog through this connection; if there is exactly one, it is selected automatically."` - CommunityID string `json:"communityId" jsonschema:"UUID of the community the Database asset (and its automatically created domain) will be created in."` - ParentSystemID string `json:"parentSystemId" jsonschema:"UUID of the parent System asset the Database asset will be linked to."` - OwnerIDs []string `json:"ownerIds" jsonschema:"UUIDs of the users to assign as owners of the Database asset. Use find_users to resolve a name (e.g. 'Admin') to its UUID."` - Description string `json:"description,omitempty" jsonschema:"Optional description of the Database asset."` - - SchemaNames []string `json:"schemaNames,omitempty" jsonschema:"Exact names of the schemas to configure for synchronization, as they appear at the data source. Required if the database exposes more than one schema; if there is exactly one, it is selected automatically. Pass the single value '*' to configure every discovered schema — only do this after confirming with the user which schemas they actually want synced, never as a silent default when multiple schemas exist."` - - Include string `json:"include" jsonschema:"Comma-separated table name pattern to synchronize, '*' wildcard supported. Required — there is no default. Confirm with the user which tables they want before calling; do not pass '*' (all tables) without asking first. Applies to every schema selected via schemaNames."` - Exclude string `json:"exclude,omitempty" jsonschema:"Optional. Comma-separated table name pattern to exclude from synchronization."` - TargetDomainID string `json:"targetDomainId,omitempty" jsonschema:"Optional. UUID of a domain to create synchronized assets in. If omitted, an automatically created domain per schema is used."` - SkipViews bool `json:"skipViews,omitempty" jsonschema:"Optional. If true, database views are excluded from synchronization."` - RegisterSourceTags bool `json:"registerSourceTags,omitempty" jsonschema:"Optional. If true, registers tags from the data source (when supported by the driver)."` - IngestSemanticViews bool `json:"ingestSemanticViews,omitempty" jsonschema:"Optional. If true, ingests semantic views (when supported by the driver)."` - RegisterDataUsageStatistics bool `json:"registerDataUsageStatistics,omitempty" jsonschema:"Optional. If true, calculates data usage statistics for the synchronized tables (when supported by the data source)."` -} - -type Output struct { - Database *clients.Database `json:"database,omitempty" jsonschema:"The registered Database asset."` - SchemaConnections []clients.SchemaConnection `json:"schemaConnections,omitempty" jsonschema:"The schema connections discovered under the database, now configured for synchronization."` - Success bool `json:"success" jsonschema:"Whether the database was fully configured."` - Error string `json:"error,omitempty" jsonschema:"Error message if configuration failed or is still in progress (safe to retry)."` -} - -func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { - return &chip.Tool[Input, Output]{ - Name: "configure_database", - Title: "Configure Database for Ingestion", - Description: "Discovers a database through an Edge connection, registers it as a Database asset, and configures which schemas/tables get synchronized. Prerequisite for start_ingestion. Assumes the target community and parent System asset already exist (create_community/create_domain/create_asset), and that a jdbc-ingestion capability referencing this connection has already been created via edge_create_capability — the discovery/refresh steps here only find data because that capability actually performs the crawl; without one, this fails with a discovery-timeout-shaped error even though the real cause is the missing capability. If more than one database or schema is discovered, or include isn't provided, this returns an error naming the candidates instead of guessing — confirm the database, schemas, and table pattern with the user, do not default to configuring everything.", - Handler: handler(collibraClient), - Permissions: []string{}, - Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, - } -} - -func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { - return func(ctx context.Context, input Input) (Output, error) { - if err := validation.UUID("edgeConnectionId", input.EdgeConnectionID); err != nil { - return Output{}, err - } - if err := validation.UUID("communityId", input.CommunityID); err != nil { - return Output{}, err - } - if err := validation.UUID("parentSystemId", input.ParentSystemID); err != nil { - return Output{}, err - } - if len(input.OwnerIDs) == 0 { - return Output{}, fmt.Errorf("ownerIds must contain at least one user UUID — use find_users to resolve a name to an id") - } - if err := validation.UUIDs("ownerIds", input.OwnerIDs); err != nil { - return Output{}, err - } - if err := validation.UUIDOptional("targetDomainId", input.TargetDomainID); err != nil { - return Output{}, err - } - if strings.TrimSpace(input.Include) == "" { - return Output{}, fmt.Errorf("include is required — confirm with the user which tables to synchronize (or that they want '*', all tables) before calling") - } - - databaseConnection, err := discoverDatabaseConnection(ctx, collibraClient, input.EdgeConnectionID, input.DatabaseName) - if err != nil { - return Output{Success: false, Error: err.Error()}, nil - } - - database, err := clients.RegisterDatabase(ctx, collibraClient, clients.AddDatabaseRequest{ - DatabaseConnectionID: databaseConnection.ID, - CommunityID: input.CommunityID, - ParentSystemID: input.ParentSystemID, - OwnerIDs: input.OwnerIDs, - Description: input.Description, - }) - if err != nil { - return Output{Success: false, Error: fmt.Sprintf("failed to register database: %s", err.Error())}, nil - } - - discoveredSchemaConnections, err := discoverSchemaConnections(ctx, collibraClient, databaseConnection.ID) - if err != nil { - return Output{Database: database, Success: false, Error: err.Error()}, nil - } - - schemaConnections, err := selectSchemaConnections(discoveredSchemaConnections, input.SchemaNames) - if err != nil { - return Output{Database: database, Success: false, Error: err.Error()}, nil - } - - configurations := make([]clients.SchemaMetadataConfiguration, len(schemaConnections)) - for i, schemaConnection := range schemaConnections { - configurations[i] = clients.SchemaMetadataConfiguration{ - SchemaConnectionID: schemaConnection.ID, - SynchronizationRules: []clients.MetadataSynchronizationRule{ - { - Include: input.Include, - Exclude: input.Exclude, - TargetDomainID: input.TargetDomainID, - SkipViews: input.SkipViews, - RegisterSourceTags: input.RegisterSourceTags, - IngestSemanticViews: input.IngestSemanticViews, - RegisterDataUsageStatistics: input.RegisterDataUsageStatistics, - }, - }, - } - } - - if _, err := clients.SetSchemaMetadataConfigurationsBatch(ctx, collibraClient, configurations); err != nil { - return Output{Database: database, SchemaConnections: schemaConnections, Success: false, - Error: fmt.Sprintf("failed to set schema synchronization rules: %s", err.Error())}, nil - } - - return Output{Database: database, SchemaConnections: schemaConnections, Success: true}, nil - } -} - -// discoverDatabaseConnection triggers a refresh of database connections for the given -// Edge connection and polls until at least one appears, then selects one by name (or -// the only one, if there's no ambiguity). -func discoverDatabaseConnection(ctx context.Context, client *http.Client, edgeConnectionID, databaseName string) (*clients.DatabaseConnection, error) { - if err := clients.RefreshDatabaseConnections(ctx, client, edgeConnectionID); err != nil { - return nil, fmt.Errorf("failed to refresh database connections: %w", err) - } - - var connections []clients.DatabaseConnection - for attempt := 0; attempt < pollAttempts; attempt++ { - if attempt > 0 { - Sleep(pollInterval) - } - found, err := clients.ListDatabaseConnections(ctx, client, edgeConnectionID) - if err != nil { - return nil, fmt.Errorf("failed to list database connections: %w", err) - } - if len(found) > 0 { - connections = found - break - } - } - - if len(connections) == 0 { - return nil, fmt.Errorf("no database connections were discovered for edge connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or — more commonly — no capability referencing this connection exists yet: the discovery/refresh only finds data because a jdbc-ingestion capability actually performs the crawl. Verify a capability exists for this connection (edge_create_capability) before retrying", edgeConnectionID, pollAttempts) - } - - if databaseName != "" { - for _, connection := range connections { - if connection.Name == databaseName { - return &connection, nil - } - } - return nil, fmt.Errorf("no database named %q found among discovered database connections for edge connection %s", databaseName, edgeConnectionID) - } - - if len(connections) > 1 { - return nil, fmt.Errorf("multiple databases discovered for edge connection %s; specify databaseName to select one", edgeConnectionID) - } - - return &connections[0], nil -} - -// discoverSchemaConnections triggers a refresh of schema connections for the given -// database connection and polls until at least one appears. -func discoverSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) ([]clients.SchemaConnection, error) { - if err := clients.RefreshSchemaConnections(ctx, client, databaseConnectionID); err != nil { - return nil, fmt.Errorf("failed to refresh schema connections: %w", err) - } - - for attempt := 0; attempt < pollAttempts; attempt++ { - if attempt > 0 { - Sleep(pollInterval) - } - found, err := clients.ListSchemaConnections(ctx, client, databaseConnectionID) - if err != nil { - return nil, fmt.Errorf("failed to list schema connections: %w", err) - } - if len(found) > 0 { - return found, nil - } - } - - return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or no capability referencing this connection exists yet — verify with edge_create_capability", databaseConnectionID, pollAttempts) -} - -// selectSchemaConnections narrows discovered to the schemas named in schemaNames. An -// empty schemaNames is only accepted when there's no ambiguity (a single discovered -// schema); with more than one discovered schema, the caller must either name the ones -// they want or pass ["*"] to explicitly opt into configuring all of them — this -// mirrors discoverDatabaseConnection's databaseName requirement, so a multi-schema -// database can never be synchronized wholesale by omission. -func selectSchemaConnections(discovered []clients.SchemaConnection, schemaNames []string) ([]clients.SchemaConnection, error) { - if len(schemaNames) == 1 && schemaNames[0] == "*" { - return discovered, nil - } - - if len(schemaNames) == 0 { - if len(discovered) > 1 { - names := make([]string, len(discovered)) - for i, s := range discovered { - names[i] = s.Name - } - return nil, fmt.Errorf("multiple schemas discovered (%s); specify schemaNames to select which to configure, or pass [\"*\"] to configure all of them", strings.Join(names, ", ")) - } - return discovered, nil - } - - byName := make(map[string]clients.SchemaConnection, len(discovered)) - for _, s := range discovered { - byName[s.Name] = s - } - - selected := make([]clients.SchemaConnection, 0, len(schemaNames)) - var missing []string - for _, name := range schemaNames { - s, ok := byName[name] - if !ok { - missing = append(missing, name) - continue - } - selected = append(selected, s) - } - if len(missing) > 0 { - return nil, fmt.Errorf("schema(s) not found among discovered schemas: %s", strings.Join(missing, ", ")) - } - - return selected, nil -} diff --git a/pkg/tools/configure_database_schemas/tool.go b/pkg/tools/configure_database_schemas/tool.go new file mode 100644 index 0000000..124c57a --- /dev/null +++ b/pkg/tools/configure_database_schemas/tool.go @@ -0,0 +1,186 @@ +// Package configure_database_schemas implements the configure_database_schemas MCP +// tool: discovers the schemas under an already-registered database and configures +// which schemas/tables get synchronized, via the public catalogDatabase API. +// +// This is the second of two steps in setting up a database for jdbc-ingestion — the +// first, registering the Database asset, is a separate tool (register_database), +// which returns the databaseConnectionId this tool takes as input. See +// register_database's package doc comment for why they're split rather than one call. +// +// Ambiguity guards: schemaNames is required when more than one schema is discovered, +// and include has no default — both force an explicit choice instead of silently +// syncing every schema/table. schemaNames/include still accept an explicit "*" to opt +// into "everything", but that must be a deliberate choice (confirmed with the user), +// never this tool's default behavior. Neither guard risks a dead end: schema discovery +// and selection have no side effects, so an ambiguity error here is a true no-op — the +// database registered by register_database is untouched, and retrying this tool call +// with the missing name(s) filled in just starts clean. +package configure_database_schemas + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// Sleep is used for polling backoff. Overridable in tests to avoid slow test runs. +var Sleep = time.Sleep + +const ( + // pollAttempts * pollInterval bounds the wait for the async refresh. A + // first-time connection to a data source can take 30-60s to establish, so + // this budget needs enough headroom to cover that cold-start case. + pollAttempts = 14 + pollInterval = 5 * time.Second +) + +type Input struct { + DatabaseConnectionID string `json:"databaseConnectionId" jsonschema:"The databaseConnectionId from register_database's Database output — identifies which registered database's schemas to configure."` + SchemaNames []string `json:"schemaNames,omitempty" jsonschema:"Exact names of the schemas to configure for synchronization, as they appear at the data source. Required if the database exposes more than one schema; if there is exactly one, it is selected automatically. Pass the single value '*' to configure every discovered schema — only do this after confirming with the user which schemas they actually want synced, never as a silent default when multiple schemas exist."` + + Include string `json:"include" jsonschema:"Comma-separated table name pattern to synchronize, '*' wildcard supported. Required — there is no default. Confirm with the user which tables they want before calling; do not pass '*' (all tables) without asking first. Applies to every schema selected via schemaNames."` + Exclude string `json:"exclude,omitempty" jsonschema:"Optional. Comma-separated table name pattern to exclude from synchronization."` + TargetDomainID string `json:"targetDomainId,omitempty" jsonschema:"Optional. UUID of a domain to create synchronized assets in. If omitted, an automatically created domain per schema is used."` + SkipViews bool `json:"skipViews,omitempty" jsonschema:"Optional. If true, database views are excluded from synchronization."` + RegisterSourceTags bool `json:"registerSourceTags,omitempty" jsonschema:"Optional. If true, registers tags from the data source (when supported by the driver)."` + IngestSemanticViews bool `json:"ingestSemanticViews,omitempty" jsonschema:"Optional. If true, ingests semantic views (when supported by the driver)."` + RegisterDataUsageStatistics bool `json:"registerDataUsageStatistics,omitempty" jsonschema:"Optional. If true, calculates data usage statistics for the synchronized tables (when supported by the data source)."` +} + +type Output struct { + SchemaConnections []clients.SchemaConnection `json:"schemaConnections,omitempty" jsonschema:"The schema connections discovered under the database, now configured for synchronization."` + Success bool `json:"success" jsonschema:"Whether the schemas were fully configured."` + Error string `json:"error,omitempty" jsonschema:"Error message if configuration failed or is still in progress (safe to retry)."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "configure_database_schemas", + Title: "Configure Database Schema Synchronization", + Description: "Discovers the schemas under a database registered via register_database and configures which schemas/tables get synchronized. Prerequisite for start_ingestion. If more than one schema is discovered, or include isn't provided, this returns an error naming the candidates instead of guessing — confirm the schemas and table pattern with the user, do not default to configuring everything. Safe to call again later to change which schemas/tables are synced.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("databaseConnectionId", input.DatabaseConnectionID); err != nil { + return Output{}, err + } + if err := validation.UUIDOptional("targetDomainId", input.TargetDomainID); err != nil { + return Output{}, err + } + if strings.TrimSpace(input.Include) == "" { + return Output{}, fmt.Errorf("include is required — confirm with the user which tables to synchronize (or that they want '*', all tables) before calling") + } + + discoveredSchemaConnections, err := discoverSchemaConnections(ctx, collibraClient, input.DatabaseConnectionID) + if err != nil { + return Output{Success: false, Error: err.Error()}, nil + } + + schemaConnections, err := selectSchemaConnections(discoveredSchemaConnections, input.SchemaNames) + if err != nil { + return Output{Success: false, Error: err.Error()}, nil + } + + configurations := make([]clients.SchemaMetadataConfiguration, len(schemaConnections)) + for i, schemaConnection := range schemaConnections { + configurations[i] = clients.SchemaMetadataConfiguration{ + SchemaConnectionID: schemaConnection.ID, + SynchronizationRules: []clients.MetadataSynchronizationRule{ + { + Include: input.Include, + Exclude: input.Exclude, + TargetDomainID: input.TargetDomainID, + SkipViews: input.SkipViews, + RegisterSourceTags: input.RegisterSourceTags, + IngestSemanticViews: input.IngestSemanticViews, + RegisterDataUsageStatistics: input.RegisterDataUsageStatistics, + }, + }, + } + } + + if _, err := clients.SetSchemaMetadataConfigurationsBatch(ctx, collibraClient, configurations); err != nil { + return Output{SchemaConnections: schemaConnections, Success: false, + Error: fmt.Sprintf("failed to set schema synchronization rules: %s", err.Error())}, nil + } + + return Output{SchemaConnections: schemaConnections, Success: true}, nil + } +} + +// discoverSchemaConnections triggers a refresh of schema connections for the given +// database connection and polls until at least one appears. +func discoverSchemaConnections(ctx context.Context, client *http.Client, databaseConnectionID string) ([]clients.SchemaConnection, error) { + if err := clients.RefreshSchemaConnections(ctx, client, databaseConnectionID); err != nil { + return nil, fmt.Errorf("failed to refresh schema connections: %w", err) + } + + for attempt := 0; attempt < pollAttempts; attempt++ { + if attempt > 0 { + Sleep(pollInterval) + } + found, err := clients.ListSchemaConnections(ctx, client, databaseConnectionID) + if err != nil { + return nil, fmt.Errorf("failed to list schema connections: %w", err) + } + if len(found) > 0 { + return found, nil + } + } + + return nil, fmt.Errorf("no schemas were discovered for database connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or no capability referencing this connection exists yet — verify with edge_create_capability", databaseConnectionID, pollAttempts) +} + +// selectSchemaConnections narrows discovered to the schemas named in schemaNames. An +// empty schemaNames is only accepted when there's no ambiguity (a single discovered +// schema); with more than one discovered schema, the caller must either name the ones +// they want or pass ["*"] to explicitly opt into configuring all of them. +func selectSchemaConnections(discovered []clients.SchemaConnection, schemaNames []string) ([]clients.SchemaConnection, error) { + if len(schemaNames) == 1 && schemaNames[0] == "*" { + return discovered, nil + } + + if len(schemaNames) == 0 { + if len(discovered) > 1 { + names := make([]string, len(discovered)) + for i, s := range discovered { + names[i] = s.Name + } + return nil, fmt.Errorf("multiple schemas discovered (%s); specify schemaNames to select which to configure, or pass [\"*\"] to configure all of them", strings.Join(names, ", ")) + } + return discovered, nil + } + + byName := make(map[string]clients.SchemaConnection, len(discovered)) + for _, s := range discovered { + byName[s.Name] = s + } + + selected := make([]clients.SchemaConnection, 0, len(schemaNames)) + var missing []string + for _, name := range schemaNames { + s, ok := byName[name] + if !ok { + missing = append(missing, name) + continue + } + selected = append(selected, s) + } + if len(missing) > 0 { + return nil, fmt.Errorf("schema(s) not found among discovered schemas: %s", strings.Join(missing, ", ")) + } + + return selected, nil +} diff --git a/pkg/tools/configure_database/tool_test.go b/pkg/tools/configure_database_schemas/tool_test.go similarity index 52% rename from pkg/tools/configure_database/tool_test.go rename to pkg/tools/configure_database_schemas/tool_test.go index 9234604..b845c49 100644 --- a/pkg/tools/configure_database/tool_test.go +++ b/pkg/tools/configure_database_schemas/tool_test.go @@ -1,4 +1,4 @@ -package configure_database_test +package configure_database_schemas_test import ( "net/http" @@ -7,7 +7,7 @@ import ( "time" "github.com/collibra/chip/pkg/clients" - tools "github.com/collibra/chip/pkg/tools/configure_database" + tools "github.com/collibra/chip/pkg/tools/configure_database_schemas" "github.com/collibra/chip/pkg/tools/testutil" "github.com/google/uuid" ) @@ -17,41 +17,10 @@ func init() { } func TestConfigureDatabase_Success(t *testing.T) { - edgeConnID, _ := uuid.NewUUID() - communityID, _ := uuid.NewUUID() - systemID, _ := uuid.NewUUID() - ownerID, _ := uuid.NewUUID() dbConnID, _ := uuid.NewUUID() - databaseID, _ := uuid.NewUUID() schemaConnID, _ := uuid.NewUUID() mux := http.NewServeMux() - mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { - if r.URL.Query().Get("edgeConnectionId") != edgeConnID.String() { - t.Fatalf("unexpected edgeConnectionId: %s", r.URL.Query().Get("edgeConnectionId")) - } - w.WriteHeader(http.StatusAccepted) - }) - mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { - return http.StatusOK, map[string]any{ - "results": []clients.DatabaseConnection{ - {ID: dbConnID.String(), Name: "source", EdgeConnectionID: edgeConnID.String()}, - }, - } - })) - mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { - if in.DatabaseConnectionID != dbConnID.String() { - t.Fatalf("unexpected databaseConnectionId: %s", in.DatabaseConnectionID) - } - return http.StatusCreated, clients.Database{ - ID: databaseID.String(), - Name: "source", - CommunityID: in.CommunityID, - OwnerIDs: in.OwnerIDs, - ParentSystemID: in.ParentSystemID, - DatabaseConnectionID: in.DatabaseConnectionID, - } - })) mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { if r.URL.Query().Get("databaseConnectionId") != dbConnID.String() { t.Fatalf("unexpected databaseConnectionId: %s", r.URL.Query().Get("databaseConnectionId")) @@ -80,11 +49,8 @@ func TestConfigureDatabase_Success(t *testing.T) { client := testutil.NewClient(server) output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ - EdgeConnectionID: edgeConnID.String(), - CommunityID: communityID.String(), - ParentSystemID: systemID.String(), - OwnerIDs: []string{ownerID.String()}, - Include: "*", + DatabaseConnectionID: dbConnID.String(), + Include: "*", }) if err != nil { t.Fatalf("expected no error, got: %v", err) @@ -92,26 +58,17 @@ func TestConfigureDatabase_Success(t *testing.T) { if !output.Success { t.Fatalf("expected success, got error: %s", output.Error) } - if output.Database.ID != databaseID.String() { - t.Fatalf("expected database id %s, got %s", databaseID.String(), output.Database.ID) - } if len(output.SchemaConnections) != 1 { t.Fatalf("expected 1 schema connection, got %d", len(output.SchemaConnections)) } } func TestConfigureDatabase_MissingInclude(t *testing.T) { - edgeConnID, _ := uuid.NewUUID() - communityID, _ := uuid.NewUUID() - systemID, _ := uuid.NewUUID() - ownerID, _ := uuid.NewUUID() + dbConnID, _ := uuid.NewUUID() client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ - EdgeConnectionID: edgeConnID.String(), - CommunityID: communityID.String(), - ParentSystemID: systemID.String(), - OwnerIDs: []string{ownerID.String()}, + DatabaseConnectionID: dbConnID.String(), }) if err == nil { t.Fatalf("expected an error when include is omitted") @@ -119,29 +76,11 @@ func TestConfigureDatabase_MissingInclude(t *testing.T) { } func TestConfigureDatabase_MultipleSchemasRequireSchemaNames(t *testing.T) { - edgeConnID, _ := uuid.NewUUID() - communityID, _ := uuid.NewUUID() - systemID, _ := uuid.NewUUID() - ownerID, _ := uuid.NewUUID() dbConnID, _ := uuid.NewUUID() - databaseID, _ := uuid.NewUUID() publicSchemaID, _ := uuid.NewUUID() privateSchemaID, _ := uuid.NewUUID() mux := http.NewServeMux() - mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusAccepted) - }) - mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { - return http.StatusOK, map[string]any{ - "results": []clients.DatabaseConnection{ - {ID: dbConnID.String(), Name: "source", EdgeConnectionID: edgeConnID.String()}, - }, - } - })) - mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { - return http.StatusCreated, clients.Database{ID: databaseID.String(), Name: "source", DatabaseConnectionID: in.DatabaseConnectionID} - })) mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) }) @@ -153,17 +92,19 @@ func TestConfigureDatabase_MultipleSchemasRequireSchemaNames(t *testing.T) { }, } })) + batchCalled := false + mux.Handle("POST /rest/catalogDatabase/v1/schemaMetadataConfigurations/batch", testutil.JsonHandlerInOut(func(r *http.Request, in []clients.SchemaMetadataConfiguration) (int, []clients.SchemaMetadataConfiguration) { + batchCalled = true + return http.StatusCreated, in + })) server := httptest.NewServer(mux) defer server.Close() client := testutil.NewClient(server) output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ - EdgeConnectionID: edgeConnID.String(), - CommunityID: communityID.String(), - ParentSystemID: systemID.String(), - OwnerIDs: []string{ownerID.String()}, - Include: "*", + DatabaseConnectionID: dbConnID.String(), + Include: "*", }) if err != nil { t.Fatalf("expected no error (failures reported via Output), got: %v", err) @@ -174,32 +115,17 @@ func TestConfigureDatabase_MultipleSchemasRequireSchemaNames(t *testing.T) { if output.Error == "" { t.Fatalf("expected an error message naming the discovered schemas") } + if batchCalled { + t.Fatalf("expected no synchronization rules to be set when schema selection fails") + } } func TestConfigureDatabase_SchemaNamesSelectsSubset(t *testing.T) { - edgeConnID, _ := uuid.NewUUID() - communityID, _ := uuid.NewUUID() - systemID, _ := uuid.NewUUID() - ownerID, _ := uuid.NewUUID() dbConnID, _ := uuid.NewUUID() - databaseID, _ := uuid.NewUUID() publicSchemaID, _ := uuid.NewUUID() privateSchemaID, _ := uuid.NewUUID() mux := http.NewServeMux() - mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusAccepted) - }) - mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { - return http.StatusOK, map[string]any{ - "results": []clients.DatabaseConnection{ - {ID: dbConnID.String(), Name: "source", EdgeConnectionID: edgeConnID.String()}, - }, - } - })) - mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { - return http.StatusCreated, clients.Database{ID: databaseID.String(), Name: "source", DatabaseConnectionID: in.DatabaseConnectionID} - })) mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) }) @@ -223,12 +149,9 @@ func TestConfigureDatabase_SchemaNamesSelectsSubset(t *testing.T) { client := testutil.NewClient(server) output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ - EdgeConnectionID: edgeConnID.String(), - CommunityID: communityID.String(), - ParentSystemID: systemID.String(), - OwnerIDs: []string{ownerID.String()}, - SchemaNames: []string{"public"}, - Include: "*", + DatabaseConnectionID: dbConnID.String(), + SchemaNames: []string{"public"}, + Include: "*", }) if err != nil { t.Fatalf("expected no error, got: %v", err) @@ -241,18 +164,15 @@ func TestConfigureDatabase_SchemaNamesSelectsSubset(t *testing.T) { } } -func TestConfigureDatabase_NoDatabaseConnectionsDiscovered(t *testing.T) { - edgeConnID, _ := uuid.NewUUID() - communityID, _ := uuid.NewUUID() - systemID, _ := uuid.NewUUID() - ownerID, _ := uuid.NewUUID() +func TestConfigureDatabase_NoSchemaConnectionsDiscovered(t *testing.T) { + dbConnID, _ := uuid.NewUUID() mux := http.NewServeMux() - mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("POST /rest/catalogDatabase/v1/schemaConnections/refresh", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) }) - mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { - return http.StatusOK, map[string]any{"results": []clients.DatabaseConnection{}} + mux.Handle("GET /rest/catalogDatabase/v1/schemaConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{"results": []clients.SchemaConnection{}} })) server := httptest.NewServer(mux) @@ -260,17 +180,14 @@ func TestConfigureDatabase_NoDatabaseConnectionsDiscovered(t *testing.T) { client := testutil.NewClient(server) output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ - EdgeConnectionID: edgeConnID.String(), - CommunityID: communityID.String(), - ParentSystemID: systemID.String(), - OwnerIDs: []string{ownerID.String()}, - Include: "*", + DatabaseConnectionID: dbConnID.String(), + Include: "*", }) if err != nil { t.Fatalf("expected no error (failures reported via Output), got: %v", err) } if output.Success { - t.Fatalf("expected failure when no database connections are discovered") + t.Fatalf("expected failure when no schema connections are discovered") } if output.Error == "" { t.Fatalf("expected an error message") @@ -280,9 +197,10 @@ func TestConfigureDatabase_NoDatabaseConnectionsDiscovered(t *testing.T) { func TestConfigureDatabase_InvalidInput(t *testing.T) { client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ - EdgeConnectionID: "not-a-uuid", + DatabaseConnectionID: "not-a-uuid", + Include: "*", }) if err == nil { - t.Fatalf("expected an error for invalid edgeConnectionId") + t.Fatalf("expected an error for invalid databaseConnectionId") } } diff --git a/pkg/tools/create_community/tool.go b/pkg/tools/create_community/tool.go index 7a3c96b..86f028d 100644 --- a/pkg/tools/create_community/tool.go +++ b/pkg/tools/create_community/tool.go @@ -1,6 +1,6 @@ // Package create_community implements the create_community MCP tool: creates a DGC // community, the top-level catalog scaffolding required before creating domains and -// registering databases via configure_database. +// registering databases via register_database. package create_community import ( @@ -28,7 +28,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "create_community", Title: "Create Community", - Description: "Creates a DGC community. A community is required before creating a domain (create_domain) and registering a database (configure_database).", + Description: "Creates a DGC community. A community is required before creating a domain (create_domain) and registering a database (register_database).", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/create_domain/tool.go b/pkg/tools/create_domain/tool.go index e32d25c..f5f4a4b 100644 --- a/pkg/tools/create_domain/tool.go +++ b/pkg/tools/create_domain/tool.go @@ -1,6 +1,6 @@ // Package create_domain implements the create_domain MCP tool: creates a DGC domain // within a community, e.g. a "Physical Data Dictionary" domain required before -// registering a database via configure_database. +// registering a database via register_database. package create_domain import ( @@ -32,7 +32,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "create_domain", Title: "Create Domain", - Description: "Creates a DGC domain within a community, e.g. a Physical Data Dictionary domain required before registering a database via configure_database.", + Description: "Creates a DGC domain within a community, e.g. a Physical Data Dictionary domain required before registering a database via register_database.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/find_users/tool.go b/pkg/tools/find_users/tool.go index 2066bf4..9edbee0 100644 --- a/pkg/tools/find_users/tool.go +++ b/pkg/tools/find_users/tool.go @@ -1,5 +1,5 @@ // Package find_users implements the find_users MCP tool: resolves a user's name -// (or partial name) to their UUID, e.g. to fill in configure_database's ownerIds +// (or partial name) to their UUID, e.g. to fill in register_database's ownerIds // without the caller needing to already know the id. package find_users @@ -39,7 +39,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "find_users", Title: "Find Users", - Description: "Finds DGC users by name (substring, case-insensitive), returning their UUIDs. Use this to resolve a name to an id for configure_database's ownerIds or similar owner/assignee fields, instead of needing to already know the UUID.", + Description: "Finds DGC users by name (substring, case-insensitive), returning their UUIDs. Use this to resolve a name to an id for register_database's ownerIds or similar owner/assignee fields, instead of needing to already know the UUID.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, diff --git a/pkg/tools/register.go b/pkg/tools/register.go index 015ef09..edf6d62 100644 --- a/pkg/tools/register.go +++ b/pkg/tools/register.go @@ -18,7 +18,7 @@ import ( "github.com/collibra/chip/pkg/tools/catalog_etl_save_generic_config" "github.com/collibra/chip/pkg/tools/catalog_etl_start_job" "github.com/collibra/chip/pkg/tools/catalog_etl_update_schedule" - "github.com/collibra/chip/pkg/tools/configure_database" + "github.com/collibra/chip/pkg/tools/configure_database_schemas" "github.com/collibra/chip/pkg/tools/create_asset" "github.com/collibra/chip/pkg/tools/create_community" "github.com/collibra/chip/pkg/tools/create_domain" @@ -45,11 +45,11 @@ import ( "github.com/collibra/chip/pkg/tools/find_users" "github.com/collibra/chip/pkg/tools/get_asset_details" "github.com/collibra/chip/pkg/tools/get_business_term_data" - "github.com/collibra/chip/pkg/tools/get_job_status" "github.com/collibra/chip/pkg/tools/get_column_semantics" "github.com/collibra/chip/pkg/tools/get_context_specification" "github.com/collibra/chip/pkg/tools/get_data_source_setup_guide" "github.com/collibra/chip/pkg/tools/get_debug_mcp_init_request" + "github.com/collibra/chip/pkg/tools/get_job_status" "github.com/collibra/chip/pkg/tools/get_lineage_downstream" "github.com/collibra/chip/pkg/tools/get_lineage_entity" "github.com/collibra/chip/pkg/tools/get_lineage_transformation" @@ -65,6 +65,7 @@ import ( "github.com/collibra/chip/pkg/tools/prepare_create_asset" "github.com/collibra/chip/pkg/tools/pull_data_contract_manifest" "github.com/collibra/chip/pkg/tools/push_data_contract_manifest" + "github.com/collibra/chip/pkg/tools/register_database" "github.com/collibra/chip/pkg/tools/remove_data_classification_match" "github.com/collibra/chip/pkg/tools/search_asset_keyword" "github.com/collibra/chip/pkg/tools/search_data_classes" @@ -137,7 +138,8 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, test_connection.NewTool(client)) toolRegister(server, toolConfig, edge_get_job_status.NewTool(client)) toolRegister(server, toolConfig, get_job_status.NewTool(client)) - toolRegister(server, toolConfig, configure_database.NewTool(client)) + toolRegister(server, toolConfig, register_database.NewTool(client)) + toolRegister(server, toolConfig, configure_database_schemas.NewTool(client)) toolRegister(server, toolConfig, start_ingestion.NewTool(client)) // Edge Management tools — read/delete/run operations with no counterpart among the diff --git a/pkg/tools/register_database/tool.go b/pkg/tools/register_database/tool.go new file mode 100644 index 0000000..a276291 --- /dev/null +++ b/pkg/tools/register_database/tool.go @@ -0,0 +1,145 @@ +// Package register_database implements the register_database MCP tool: discovers a +// database through an Edge connection and registers it as a DGC Database asset, via +// the public catalogDatabase API. +// +// This is the first of two steps in setting up a database for jdbc-ingestion — the +// second, configuring which schemas/tables get synchronized, is a separate tool +// (configure_database_schemas), which takes this tool's returned databaseConnectionId. They +// were originally one tool; split apart because registering the database is a +// one-time, id-claiming operation (POST /databases permanently claims the underlying +// database connection — a second call for the same one fails with +// databaseConnectionAlreadyUsed) while schema configuration is something an agent may +// reasonably want to redo independently, and keeping them separate means a +// schema-selection ambiguity error here never risks colliding with a database this +// tool already registered. +package register_database + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/validation" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// Sleep is used for polling backoff. Overridable in tests to avoid slow test runs. +var Sleep = time.Sleep + +const ( + // pollAttempts * pollInterval bounds the wait for the async refresh. A + // first-time connection to a data source can take 30-60s to establish, so + // this budget needs enough headroom to cover that cold-start case. + pollAttempts = 14 + pollInterval = 5 * time.Second +) + +type Input struct { + EdgeConnectionID string `json:"edgeConnectionId" jsonschema:"UUID of the Edge connection (created via edge_create_connection) to discover a database on."` + DatabaseName string `json:"databaseName,omitempty" jsonschema:"Optional. Exact name of the database (catalog) to register, as it appears at the data source. Required if the data source exposes more than one database/catalog through this connection; if there is exactly one, it is selected automatically."` + CommunityID string `json:"communityId" jsonschema:"UUID of the community the Database asset (and its automatically created domain) will be created in."` + ParentSystemID string `json:"parentSystemId" jsonschema:"UUID of the parent System asset the Database asset will be linked to."` + OwnerIDs []string `json:"ownerIds" jsonschema:"UUIDs of the users to assign as owners of the Database asset. Use find_users to resolve a name (e.g. 'Admin') to its UUID."` + Description string `json:"description,omitempty" jsonschema:"Optional description of the Database asset."` +} + +type Output struct { + Database *clients.Database `json:"database,omitempty" jsonschema:"The registered Database asset. Pass its databaseConnectionId to configure_database_schemas next."` + Success bool `json:"success" jsonschema:"Whether the database was registered successfully."` + Error string `json:"error,omitempty" jsonschema:"Error message if registration failed or is still in progress (safe to retry)."` +} + +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "register_database", + Title: "Register Database for Ingestion", + Description: "Discovers a database through an Edge connection and registers it as a Database asset. Prerequisite for configure_database_schemas (which sets up schema/table synchronization) and, transitively, start_ingestion. Assumes the target community and parent System asset already exist (create_community/create_domain/create_asset), and that a jdbc-ingestion capability referencing this connection has already been created via edge_create_capability — discovery only finds data because that capability actually performs the crawl; without one, this fails with a discovery-timeout-shaped error even though the real cause is the missing capability. If more than one database is discovered, this returns an error naming the candidates instead of guessing — confirm which one with the user.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if err := validation.UUID("edgeConnectionId", input.EdgeConnectionID); err != nil { + return Output{}, err + } + if err := validation.UUID("communityId", input.CommunityID); err != nil { + return Output{}, err + } + if err := validation.UUID("parentSystemId", input.ParentSystemID); err != nil { + return Output{}, err + } + if len(input.OwnerIDs) == 0 { + return Output{}, fmt.Errorf("ownerIds must contain at least one user UUID — use find_users to resolve a name to an id") + } + if err := validation.UUIDs("ownerIds", input.OwnerIDs); err != nil { + return Output{}, err + } + + databaseConnection, err := discoverDatabaseConnection(ctx, collibraClient, input.EdgeConnectionID, input.DatabaseName) + if err != nil { + return Output{Success: false, Error: err.Error()}, nil + } + + database, err := clients.RegisterDatabase(ctx, collibraClient, clients.AddDatabaseRequest{ + DatabaseConnectionID: databaseConnection.ID, + CommunityID: input.CommunityID, + ParentSystemID: input.ParentSystemID, + OwnerIDs: input.OwnerIDs, + Description: input.Description, + }) + if err != nil { + return Output{Success: false, Error: fmt.Sprintf("failed to register database: %s", err.Error())}, nil + } + + return Output{Database: database, Success: true}, nil + } +} + +// discoverDatabaseConnection triggers a refresh of database connections for the given +// Edge connection and polls until at least one appears, then selects one by name (or +// the only one, if there's no ambiguity). +func discoverDatabaseConnection(ctx context.Context, client *http.Client, edgeConnectionID, databaseName string) (*clients.DatabaseConnection, error) { + if err := clients.RefreshDatabaseConnections(ctx, client, edgeConnectionID); err != nil { + return nil, fmt.Errorf("failed to refresh database connections: %w", err) + } + + var connections []clients.DatabaseConnection + for attempt := 0; attempt < pollAttempts; attempt++ { + if attempt > 0 { + Sleep(pollInterval) + } + found, err := clients.ListDatabaseConnections(ctx, client, edgeConnectionID) + if err != nil { + return nil, fmt.Errorf("failed to list database connections: %w", err) + } + if len(found) > 0 { + connections = found + break + } + } + + if len(connections) == 0 { + return nil, fmt.Errorf("no database connections were discovered for edge connection %s after %d attempts. Either the refresh is still in progress (retry this tool call), or — more commonly — no capability referencing this connection exists yet: the discovery/refresh only finds data because a jdbc-ingestion capability actually performs the crawl. Verify a capability exists for this connection (edge_create_capability) before retrying", edgeConnectionID, pollAttempts) + } + + if databaseName != "" { + for _, connection := range connections { + if connection.Name == databaseName { + return &connection, nil + } + } + return nil, fmt.Errorf("no database named %q found among discovered database connections for edge connection %s", databaseName, edgeConnectionID) + } + + if len(connections) > 1 { + return nil, fmt.Errorf("multiple databases discovered for edge connection %s; specify databaseName to select one", edgeConnectionID) + } + + return &connections[0], nil +} diff --git a/pkg/tools/register_database/tool_test.go b/pkg/tools/register_database/tool_test.go new file mode 100644 index 0000000..3386fb8 --- /dev/null +++ b/pkg/tools/register_database/tool_test.go @@ -0,0 +1,188 @@ +package register_database_test + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/collibra/chip/pkg/clients" + tools "github.com/collibra/chip/pkg/tools/register_database" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/google/uuid" +) + +func init() { + tools.Sleep = func(time.Duration) {} +} + +func TestRegisterDatabase_Success(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + dbConnID, _ := uuid.NewUUID() + databaseID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("edgeConnectionId") != edgeConnID.String() { + t.Fatalf("unexpected edgeConnectionId: %s", r.URL.Query().Get("edgeConnectionId")) + } + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.DatabaseConnection{ + {ID: dbConnID.String(), Name: "source", EdgeConnectionID: edgeConnID.String()}, + }, + } + })) + mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { + if in.DatabaseConnectionID != dbConnID.String() { + t.Fatalf("unexpected databaseConnectionId: %s", in.DatabaseConnectionID) + } + return http.StatusCreated, clients.Database{ + ID: databaseID.String(), + Name: "source", + CommunityID: in.CommunityID, + OwnerIDs: in.OwnerIDs, + ParentSystemID: in.ParentSystemID, + DatabaseConnectionID: in.DatabaseConnectionID, + } + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err != nil { + t.Fatalf("expected no error, got: %v", err) + } + if !output.Success { + t.Fatalf("expected success, got error: %s", output.Error) + } + if output.Database.ID != databaseID.String() { + t.Fatalf("expected database id %s, got %s", databaseID.String(), output.Database.ID) + } + if output.Database.DatabaseConnectionID != dbConnID.String() { + t.Fatalf("expected databaseConnectionId %s, got %s", dbConnID.String(), output.Database.DatabaseConnectionID) + } +} + +func TestRegisterDatabase_MultipleDatabasesRequireDatabaseName(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + dbConnID1, _ := uuid.NewUUID() + dbConnID2, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{ + "results": []clients.DatabaseConnection{ + {ID: dbConnID1.String(), Name: "source1", EdgeConnectionID: edgeConnID.String()}, + {ID: dbConnID2.String(), Name: "source2", EdgeConnectionID: edgeConnID.String()}, + }, + } + })) + databaseRegistered := false + mux.Handle("POST /rest/catalogDatabase/v1/databases", testutil.JsonHandlerInOut(func(r *http.Request, in clients.AddDatabaseRequest) (int, clients.Database) { + databaseRegistered = true + return http.StatusCreated, clients.Database{DatabaseConnectionID: in.DatabaseConnectionID} + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err != nil { + t.Fatalf("expected no error (failures reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when multiple databases are discovered without databaseName") + } + if output.Error == "" { + t.Fatalf("expected an error message naming the discovered databases") + } + if databaseRegistered { + t.Fatalf("expected no Database asset to be registered when database selection is ambiguous") + } +} + +func TestRegisterDatabase_NoDatabaseConnectionsDiscovered(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + ownerID, _ := uuid.NewUUID() + + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/catalogDatabase/v1/databaseConnections/refresh", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + }) + mux.Handle("GET /rest/catalogDatabase/v1/databaseConnections", testutil.JsonHandlerOut(func(r *http.Request) (int, map[string]any) { + return http.StatusOK, map[string]any{"results": []clients.DatabaseConnection{}} + })) + + server := httptest.NewServer(mux) + defer server.Close() + + client := testutil.NewClient(server) + output, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + OwnerIDs: []string{ownerID.String()}, + }) + if err != nil { + t.Fatalf("expected no error (failures reported via Output), got: %v", err) + } + if output.Success { + t.Fatalf("expected failure when no database connections are discovered") + } + if output.Error == "" { + t.Fatalf("expected an error message") + } +} + +func TestRegisterDatabase_InvalidInput(t *testing.T) { + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: "not-a-uuid", + }) + if err == nil { + t.Fatalf("expected an error for invalid edgeConnectionId") + } +} + +func TestRegisterDatabase_RequiresOwnerIDs(t *testing.T) { + edgeConnID, _ := uuid.NewUUID() + communityID, _ := uuid.NewUUID() + systemID, _ := uuid.NewUUID() + + client := testutil.NewClient(httptest.NewServer(http.NewServeMux())) + _, err := tools.NewTool(client).Handler(t.Context(), tools.Input{ + EdgeConnectionID: edgeConnID.String(), + CommunityID: communityID.String(), + ParentSystemID: systemID.String(), + }) + if err == nil { + t.Fatalf("expected an error when ownerIds is empty") + } +} diff --git a/pkg/tools/start_ingestion/tool.go b/pkg/tools/start_ingestion/tool.go index aa2b263..5b02410 100644 --- a/pkg/tools/start_ingestion/tool.go +++ b/pkg/tools/start_ingestion/tool.go @@ -14,8 +14,8 @@ import ( ) type Input struct { - DatabaseID string `json:"databaseId" jsonschema:"UUID of the Database asset to synchronize (as registered by configure_database)."` - SchemaConnectionIDs []string `json:"schemaConnectionIds,omitempty" jsonschema:"Optional. UUIDs of specific schema connections to synchronize. If omitted, all schemas that already have synchronization rules configured (via configure_database) are synchronized."` + DatabaseID string `json:"databaseId" jsonschema:"UUID of the Database asset to synchronize (as registered by register_database)."` + SchemaConnectionIDs []string `json:"schemaConnectionIds,omitempty" jsonschema:"Optional. UUIDs of specific schema connections to synchronize. If omitted, all schemas that already have synchronization rules configured (via configure_database_schemas) are synchronized."` } type Output struct { @@ -28,7 +28,7 @@ func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { return &chip.Tool[Input, Output]{ Name: "start_ingestion", Title: "Start Database Ingestion", - Description: "Triggers the jdbc-ingestion capability run for a registered Database asset, synchronizing its configured schemas/tables into the catalog. The database must already be registered and configured via configure_database. Poll the returned job's id with get_job_status to confirm it actually completed — a 202/success response here only means the job was accepted, not that ingestion finished.", + Description: "Triggers the jdbc-ingestion capability run for a registered Database asset, synchronizing its configured schemas/tables into the catalog. The database must already be registered (register_database) and have its schemas configured (configure_database_schemas). Poll the returned job's id with get_job_status to confirm it actually completed — a 202/success response here only means the job was accepted, not that ingestion finished.", Handler: handler(collibraClient), Permissions: []string{}, Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(true)}, diff --git a/pkg/tools/upload_file/tool.go b/pkg/tools/upload_file/tool.go index 7295b29..7387707 100644 --- a/pkg/tools/upload_file/tool.go +++ b/pkg/tools/upload_file/tool.go @@ -49,9 +49,10 @@ // catalog-backed driver API (POST /rest/edge/v1/drivers/jars), which has no UI in // front of it at all, and the frontend's only file-upload path is the legacy one // embedded inside the connection-creation form's own FILE-type field. Once the user -// confirms the connection is saved, use edge_find_connections (by name) to pick up its id -// and proceed with edge_create_capability/configure_database/start_ingestion against it — -// skip edge_create_connection entirely for that connection, since it already exists. +// confirms the connection is saved, use edge_find_connections (by name) to pick up its +// id and proceed with edge_create_capability/register_database/ +// configure_database_schemas/start_ingestion against it — skip edge_create_connection +// entirely for that connection, since it already exists. package upload_file import ( From 958f842b23f5fb12bd585442d91143604bcd9422 Mon Sep 17 00:00:00 2001 From: "volodymyr.bodnar" Date: Wed, 8 Jul 2026 13:42:50 -0400 Subject: [PATCH 10/10] docs(etl-skill): document mandatory techlinHost/techlinKey custom params for lineage capabilities The Databricks and Dataplex lineage capabilities push to the Collibra Data Lineage (Techlin) server and require techlinHost/techlinKey custom parameters to be set before the first run. - databricks-lineage.md: add mandatory customParameters (techlinHost/techlinKey) to Capability parameters + gotcha. - dataplex-lineage.md: add Capability parameters section with the same mandatory custom params + gotcha. - thoughtspot.md: document explicit type / secret requirement for direct (non-DGC-routed) Techlin submission. --- .../etl-integration/references/databricks-lineage.md | 5 ++++- .../etl-integration/references/dataplex-lineage.md | 8 ++++++++ .../collibra/etl-integration/references/thoughtspot.md | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md b/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md index 12a9ff7..8ad4655 100644 --- a/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md +++ b/pkg/skills/files/collibra/etl-integration/references/databricks-lineage.md @@ -17,7 +17,9 @@ Extracts data lineage from the Databricks Unity Catalog lineage system tables an - `sourceId` (required) — unique lineage source id; also the key for the exclusion flow. - `processingLevel` — `Analyze` (load + analyze) or `Sync` (also synchronize to DGC). - `timeFrame` — lookback window in days (default 365). - +- **`customParameters` (mandatory)** — a user-defined list that must include the Techlin server coordinates: + - `techlinHost` — the Techlin/Data Lineage server host URL for this environment (e.g. `https://techlin-.cp.collibra-ops.com`). + - `techlinKey` — the Techlin API key. ## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) - `system` (required) — target System asset the ingested objects are stitched to. - `includeFilters` / `excludeFilters` — scope by `Workspace > Catalog > Schema` (wildcards `*`/`?`; exclude wins). @@ -30,3 +32,4 @@ Extracts data lineage from the Databricks Unity Catalog lineage system tables an - Set `useCollibrasystemName` when the same paths exist across environments (e.g. PROD/DEV), otherwise stitching is non-deterministic; changing it requires re-analyzing all sources. - Requires read permissions on the Unity Catalog lineage system tables (`system.access.column_lineage`, `system.access.table_lineage`, `system.query.history`) plus compute attach rights. - Lineage retention is fixed at 365 days by Databricks; large datasets rely on the default `scalable` query strategy (tune via `jvm-args`). +- **`techlinHost`/`techlinKey` custom parameters are mandatory** — set them before the first run. See the `customParameters` entry under "Capability parameters". diff --git a/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md b/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md index f21fb19..4baa975 100644 --- a/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md +++ b/pkg/skills/files/collibra/etl-integration/references/dataplex-lineage.md @@ -16,6 +16,13 @@ Extracts technical lineage from Google Dataplex (Data Lineage API) plus BigQuery - **Table lineage** (`lineageType` default) — table-level lineage; supports `includeFilters` / `excludeFilters`. - **Column lineage** — column-level lineage; requires `gcsBucket` and the (preview) Export Metadata API. Note: stitching is not supported at table level, only column level. +## Capability parameters (set on `edge_create_capability`) +- `active` — set `false` for a `sourceId` to remove that source's lineage from Techlin (there is no outbound flow). +- `useCollibrasystemName` — see gotchas; enable when identical paths exist across environments. +- **`customParameters` (mandatory)** — a user-defined list that must include the Techlin server coordinates: + - `techlinHost` — the Techlin/Data Lineage server host URL for this environment (e.g. `https://techlin-.cp.collibra-ops.com`). + - `techlinKey` — the Techlin API key. + ## Notable generic-config choices (fetch exact schema via `catalog_etl_get_schema`) - `system` (required) — target System asset the ingested objects are stitched to. - `projects` — GCP project IDs where Dataplex lineage is enabled (required for WIF; else derived from the SA key). @@ -30,3 +37,4 @@ Extracts technical lineage from Google Dataplex (Data Lineage API) plus BigQuery - Enable `useCollibrasystemName` (capability param) when identical paths exist across environments (PROD/DEV) — otherwise stitching is non-deterministic. Changing it requires re-analyzing all sources. - Dataplex retains lineage for only 30 days; older lineage is permanently gone, so runs must be scheduled recurringly. - Column lineage depends on the preview `exportMetadata` API, which must be enabled by Google on request. +- **`techlinHost`/`techlinKey` custom parameters are mandatory** — set them before the first run. See the `customParameters` entry under "Capability parameters". diff --git a/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md b/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md index d2f073c..a7c4209 100644 --- a/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md +++ b/pkg/skills/files/collibra/etl-integration/references/thoughtspot.md @@ -22,7 +22,7 @@ Extracts metadata from a ThoughtSpot cloud tenant — Tenant, Orgs, Connections, - `sourceId` (required) — stable string identifier used to construct the lin-sdk source name (`ThoughtSpot--`). Changing this across runs creates a new parallel lineage source instead of updating the existing one. - `orgs` (optional) — same org-filter semantics as the sync tab; should be aligned with the sync configuration to avoid partial graphs. -The lineage capability also requires these **install-time** run-parameters (not in the generic config): `targetDgc` (DGC URL), `username` and `password` (Collibra DGC credentials for lin-sdk — distinct from the ThoughtSpot connection credentials). Optionally `customParameters.techlinHost` + `customParameters.techlinKey` if bypassing DGC-routed lineage submission. +The lineage capability also requires these **install-time** run-parameters (not in the generic config): `targetDgc` (DGC URL), `username` and `password` (Collibra DGC credentials for lin-sdk — distinct from the ThoughtSpot connection credentials). Optionally `customParameters.techlinHost` + `customParameters.techlinKey` if bypassing DGC-routed lineage submission and pushing to the Techlin server directly. ## Gotchas