diff --git a/.agent/skills/pr-checklist/SKILL.md b/.agent/skills/pr-checklist/SKILL.md index 86559354fcb..e01058a0120 100644 --- a/.agent/skills/pr-checklist/SKILL.md +++ b/.agent/skills/pr-checklist/SKILL.md @@ -16,8 +16,9 @@ Before submitting a PR, run these commands to match what CI checks. CI uses the # 3. Tests (CI runs with both deployment engines) ./task test -# 4. If you changed bundle config structs or schema-related code: +# 4. If you changed bundle config structs, schema, or direct-engine resource code: ./task generate-schema +./task generate-direct # 5. If you changed files in python/: ./task pydabs-codegen pydabs-test pydabs-lint pydabs-docs diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index c58fd3ffb13..f8eab452b24 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -12,5 +12,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) * Support `replace_existing: true` on `postgres_branches` and `postgres_endpoints` so bundles can manage the implicitly-created production branch and primary read-write endpoint of a Lakebase project. +* Add `postgres_catalogs` resource to bind a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch ([#5265](https://github.com/databricks/cli/pull/5265)). ### Dependency updates diff --git a/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl new file mode 100644 index 00000000000..2ae10852504 --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + postgres_projects: + project: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + + postgres_catalogs: + catalog: + catalog_id: test_pg_catalog_$UNIQUE_NAME + branch: ${resources.postgres_projects.project.name}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 257e33005a3..abfb10c5f73 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -44,6 +44,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", @@ -67,6 +68,7 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"] +no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_catalog.yml.tmpl"] # External locations require actual storage credentials with cloud IAM setup # which are environment-specific, so we only test locally with the mock server diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index b9b2cb3e35c..d50035a048f 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2726,6 +2726,24 @@ resources.postgres_branches.*.ttl *duration.Duration ALL resources.postgres_branches.*.uid string REMOTE resources.postgres_branches.*.update_time *time.Time REMOTE resources.postgres_branches.*.url string INPUT +resources.postgres_catalogs.*.branch string ALL +resources.postgres_catalogs.*.catalog_id string ALL +resources.postgres_catalogs.*.create_database_if_missing bool ALL +resources.postgres_catalogs.*.create_time *time.Time REMOTE +resources.postgres_catalogs.*.id string INPUT +resources.postgres_catalogs.*.lifecycle resources.Lifecycle INPUT +resources.postgres_catalogs.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_catalogs.*.modified_status string INPUT +resources.postgres_catalogs.*.name string REMOTE +resources.postgres_catalogs.*.postgres_database string ALL +resources.postgres_catalogs.*.status *postgres.CatalogCatalogStatus REMOTE +resources.postgres_catalogs.*.status.branch string REMOTE +resources.postgres_catalogs.*.status.catalog_id string REMOTE +resources.postgres_catalogs.*.status.postgres_database string REMOTE +resources.postgres_catalogs.*.status.project string REMOTE +resources.postgres_catalogs.*.uid string REMOTE +resources.postgres_catalogs.*.update_time *time.Time REMOTE +resources.postgres_catalogs.*.url string INPUT resources.postgres_endpoints.*.autoscaling_limit_max_cu float64 ALL resources.postgres_endpoints.*.autoscaling_limit_min_cu float64 ALL resources.postgres_endpoints.*.create_time *time.Time REMOTE diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..b339f4c1354 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: deploy-postgres-catalog-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Catalog" + pg_version: 16 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.json b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.json new file mode 100644 index 00000000000..891733f4150 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.json @@ -0,0 +1,31 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Catalog", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "lakebase_test_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/catalogs/lakebase_test_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/output.txt b/acceptance/bundle/resources/postgres_catalogs/basic/output.txt new file mode 100644 index 00000000000..7bf45891cb3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/output.txt @@ -0,0 +1,74 @@ + +>>> [CLI] bundle validate +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default +Resources: + Postgres catalogs: + my_catalog: + Name: lakebase_test_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME] + Postgres projects: + my_project: + Name: Test Project for Catalog + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] postgres get-catalog catalogs/lakebase_test_[UNIQUE_NAME] +{ + "name": "catalogs/lakebase_test_[UNIQUE_NAME]", + "status": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "catalog_id": "lakebase_test_[UNIQUE_NAME]", + "postgres_database": "appdb", + "project": "projects/test-pg-proj-[UNIQUE_NAME]" + } +} + +>>> [CLI] bundle summary +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default +Resources: + Postgres catalogs: + my_catalog: + Name: lakebase_test_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME] + Postgres projects: + my_project: + Name: Test Project for Catalog + URL: (not deployed) + +>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/script b/acceptance/bundle/resources/postgres_catalogs/basic/script new file mode 100644 index 00000000000..ba6240838c8 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +trace $CLI bundle summary + +rm -f out.requests.txt +trace $CLI bundle deploy + +# Get catalog details. Hide volatile fields so cloud and local match. +catalog_name="catalogs/lakebase_test_${UNIQUE_NAME}" +trace $CLI postgres get-catalog "${catalog_name}" | jq 'del(.create_time, .update_time, .uid)' + +trace $CLI bundle summary + +# Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..bf3c2b56576 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: recreate-postgres-catalog-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Catalog Recreate" + pg_version: 16 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: $POSTGRES_DATABASE + create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt b/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt new file mode 100644 index 00000000000..1f6bb621987 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt @@ -0,0 +1,31 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +recreate postgres_catalogs.my_catalog + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/script b/acceptance/bundle/resources/postgres_catalogs/recreate/script new file mode 100644 index 00000000000..4d9de784299 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +export POSTGRES_DATABASE=appdb +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy + +# Toggle a recreate-on-change field; plan must show delete + create. +export POSTGRES_DATABASE=otherdb +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged" + +trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_catalogs/test.toml b/acceptance/bundle/resources/postgres_catalogs/test.toml new file mode 100644 index 00000000000..66f0811b343 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/test.toml @@ -0,0 +1,26 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +# Lakebase v2 (postgres) is only available in AWS as of January 2026 +CloudEnvs.gcp = false +CloudEnvs.azure = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Clean up ?o= suffix after URL since not all workspaces have that +Old = '\?o=\[(NUMID|ALPHANUMID)\]' +New = '' +Order = 1000 + +[[Repls]] +# Normalize postgres operation IDs (unique per operation). +Old = '/operations/[A-Za-z0-9+/=-]+' +New = '/operations/[OPERATION_ID]' +Order = 2000 diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index e472241f282..d78c1ad8145 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -28,6 +28,7 @@ var unsupportedResources = []string{ "synced_database_tables", "postgres_branches", "postgres_endpoints", + "postgres_catalogs", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index fe9c9a1db06..927c7a19132 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -247,6 +247,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "postgres_catalog1": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "postgres_catalog_1", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "vs_endpoint1": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -441,6 +448,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { "PostgresProjects", "PostgresBranches", "PostgresEndpoints", + "PostgresCatalogs", } diags := bundle.ApplySeq(t.Context(), b, ApplyTargetMode(), ApplyPresets()) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 0b7003f5873..58b27113a52 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -46,6 +46,7 @@ func allResourceTypes(t *testing.T) []string { "models", "pipelines", "postgres_branches", + "postgres_catalogs", "postgres_endpoints", "postgres_projects", "quality_monitors", @@ -174,6 +175,7 @@ var allowList = []string{ "pipelines", "models", "postgres_branches", + "postgres_catalogs", "postgres_endpoints", "postgres_projects", "registered_models", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 225ec32165d..30a7f4e95e5 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -35,6 +35,7 @@ type Resources struct { PostgresProjects map[string]*resources.PostgresProject `json:"postgres_projects,omitempty"` PostgresBranches map[string]*resources.PostgresBranch `json:"postgres_branches,omitempty"` PostgresEndpoints map[string]*resources.PostgresEndpoint `json:"postgres_endpoints,omitempty"` + PostgresCatalogs map[string]*resources.PostgresCatalog `json:"postgres_catalogs,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` } @@ -112,6 +113,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_projects"], r.PostgresProjects), collectResourceMap(descriptions["postgres_branches"], r.PostgresBranches), collectResourceMap(descriptions["postgres_endpoints"], r.PostgresEndpoints), + collectResourceMap(descriptions["postgres_catalogs"], r.PostgresCatalogs), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), } } @@ -167,6 +169,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), + "postgres_catalogs": (&resources.PostgresCatalog{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), } } diff --git a/bundle/config/resources/postgres_catalog.go b/bundle/config/resources/postgres_catalog.go new file mode 100644 index 00000000000..c17ffaa1682 --- /dev/null +++ b/bundle/config/resources/postgres_catalog.go @@ -0,0 +1,66 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type PostgresCatalogConfig struct { + postgres.CatalogCatalogSpec + + // CatalogId is the user-specified UC catalog name. Becomes the trailing + // component of the server-assigned Name: "catalogs/{catalog_id}". + CatalogId string `json:"catalog_id"` +} + +func (c *PostgresCatalogConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c *PostgresCatalogConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type PostgresCatalog struct { + BaseResource + PostgresCatalogConfig +} + +func (c *PostgresCatalog) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: name}) + if err != nil { + log.Debugf(ctx, "postgres catalog %s does not exist", name) + return false, err + } + return true, nil +} + +func (c *PostgresCatalog) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "postgres_catalog", + PluralName: "postgres_catalogs", + SingularTitle: "Postgres catalog", + PluralTitle: "Postgres catalogs", + } +} + +func (c *PostgresCatalog) GetName() string { + return c.CatalogId +} + +func (c *PostgresCatalog) GetURL() string { + return c.URL +} + +func (c *PostgresCatalog) InitializeURL(baseURL url.URL) { + if c.CatalogId == "" { + return + } + baseURL.Path = "explore/data/" + c.CatalogId + c.URL = baseURL.String() +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 943b279a288..626fac07983 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -273,6 +273,13 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "my_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "my_postgres_catalog", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -312,6 +319,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetProject(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetBranch(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockPostgresAPI().EXPECT().GetCatalog(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) allResources := supportedResources.AllResources() diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index fdcb671bdd3..7a1acb3d4e6 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -16,7 +16,7 @@ type interpolateMutator struct{} // Postgres resources use "name" instead of "id" as their identifier attribute. func isPostgresResource(resourceType string) bool { switch resourceType { - case "postgres_projects", "postgres_branches", "postgres_endpoints": + case "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs": return true default: return false diff --git a/bundle/deploy/terraform/pkg.go b/bundle/deploy/terraform/pkg.go index a66e5cb6a06..ff68c1b1513 100644 --- a/bundle/deploy/terraform/pkg.go +++ b/bundle/deploy/terraform/pkg.go @@ -129,6 +129,7 @@ var GroupToTerraformName = map[string]string{ "postgres_projects": "databricks_postgres_project", "postgres_branches": "databricks_postgres_branch", "postgres_endpoints": "databricks_postgres_endpoint", + "postgres_catalogs": "databricks_postgres_catalog", // 3 level groups: resources.*.GROUP "permissions": "databricks_permissions", diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go new file mode 100644 index 00000000000..b84fb09d717 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go @@ -0,0 +1,56 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +type postgresCatalogConverter struct{} + +func (c postgresCatalogConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + // The bundle config has flattened CatalogCatalogSpec fields at the top level. + // Terraform expects them nested in a "spec" block. + specFields := specFieldNames(schema.ResourcePostgresCatalogSpec{}) + topLevelFields := []string{"catalog_id"} + + specMap := make(map[string]dyn.Value) + for _, field := range specFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + specMap[field] = v + } + } + + outMap := make(map[string]dyn.Value) + for _, field := range topLevelFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + outMap[field] = v + } + } + if len(specMap) > 0 { + outMap["spec"] = dyn.V(specMap) + } + + vout := dyn.V(outMap) + + vout, diags := convert.Normalize(schema.ResourcePostgresCatalog{}, vout) + for _, diag := range diags { + log.Debugf(ctx, "postgres catalog normalization diagnostic: %s", diag.Summary) + } + + vout, err := convertLifecycle(ctx, vout, vin.Get("lifecycle")) + if err != nil { + return err + } + + out.PostgresCatalog[key] = vout.AsAny() + + return nil +} + +func init() { + registerConverter("postgres_catalogs", postgresCatalogConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go new file mode 100644 index 00000000000..8eca766f613 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go @@ -0,0 +1,73 @@ +package tfdyn + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertPostgresCatalog(t *testing.T) { + src := resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "shop_lakebase", + CatalogCatalogSpec: postgres.CatalogCatalogSpec{ + Branch: "projects/my-shop/branches/production", + PostgresDatabase: "appdb", + CreateDatabaseIfMissing: true, + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresCatalogConverter{}.Convert(ctx, "my_postgres_catalog", vin, out) + require.NoError(t, err) + + postgresCatalog := out.PostgresCatalog["my_postgres_catalog"] + assert.Equal(t, map[string]any{ + "catalog_id": "shop_lakebase", + "spec": map[string]any{ + "branch": "projects/my-shop/branches/production", + "postgres_database": "appdb", + "create_database_if_missing": true, + }, + }, postgresCatalog) +} + +func TestConvertPostgresCatalogMinimal(t *testing.T) { + src := resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "minimal_catalog", + CatalogCatalogSpec: postgres.CatalogCatalogSpec{ + Branch: "projects/my-shop/branches/production", + PostgresDatabase: "appdb", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresCatalogConverter{}.Convert(ctx, "minimal_postgres_catalog", vin, out) + require.NoError(t, err) + + postgresCatalog := out.PostgresCatalog["minimal_postgres_catalog"] + assert.Equal(t, map[string]any{ + "catalog_id": "minimal_catalog", + "spec": map[string]any{ + "branch": "projects/my-shop/branches/production", + "postgres_database": "appdb", + }, + }, postgresCatalog) +} diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 632d32bca19..69c3c4f886d 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -96,7 +96,7 @@ func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap // The direct engine manages permissions as a sub-resource // (SecretScopeFixups adds MANAGE ACL for the current user). result[resourceKey+".permissions"] = ResourceState{ID: instance.Attributes.Name} - case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints": + case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs": resourceKey = "resources." + groupName + "." + resource.Name resourceState = ResourceState{ID: instance.Attributes.Name} case "dashboards": diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ddc30c41f54..497bf9bcb7a 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -23,6 +23,7 @@ var SupportedResources = map[string]any{ "postgres_projects": (*ResourcePostgresProject)(nil), "postgres_branches": (*ResourcePostgresBranch)(nil), "postgres_endpoints": (*ResourcePostgresEndpoint)(nil), + "postgres_catalogs": (*ResourcePostgresCatalog)(nil), "alerts": (*ResourceAlert)(nil), "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 2c0a2e52f22..f130afc6194 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -191,6 +191,12 @@ var testConfig map[string]any = map[string]any{ }, }, + "postgres_catalogs": &resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + "alerts": &resources.Alert{ AlertV2: sql.AlertV2{ DisplayName: "my-alert", diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index a80b3baa69b..6478f029feb 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -28,6 +28,8 @@ pipelines: pipelines.CreatePipeline postgres_branches: postgres.BranchSpec +postgres_catalogs: postgres.CatalogCatalogSpec + postgres_endpoints: postgres.EndpointSpec postgres_projects: postgres.ProjectStatus diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index 29db9b67b20..c37dfbccbb1 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -9,3 +9,5 @@ postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec postgres_projects: postgres.ProjectSpec + +postgres_catalogs: postgres.CatalogCatalogSpec diff --git a/bundle/direct/dresources/postgres_catalog.go b/bundle/direct/dresources/postgres_catalog.go new file mode 100644 index 00000000000..48a3262e4ad --- /dev/null +++ b/bundle/direct/dresources/postgres_catalog.go @@ -0,0 +1,133 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +// PostgresCatalogRemote is the return type for DoRead. It embeds CatalogCatalogSpec so +// that all paths in StateType are valid paths in RemoteType, enabling drift detection +// for spec fields once the backend echoes spec on GET. +type PostgresCatalogRemote struct { + postgres.CatalogCatalogSpec + + CatalogId string `json:"catalog_id,omitempty"` + + Name string `json:"name,omitempty"` + Status *postgres.CatalogCatalogStatus `json:"status,omitempty"` + Uid string `json:"uid,omitempty"` + CreateTime *sdktime.Time `json:"create_time,omitempty"` + UpdateTime *sdktime.Time `json:"update_time,omitempty"` +} + +// Custom marshaler needed because embedded CatalogCatalogSpec has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *PostgresCatalogRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s PostgresCatalogRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +type ResourcePostgresCatalog struct { + client *databricks.WorkspaceClient +} + +type PostgresCatalogState = resources.PostgresCatalogConfig + +func (*ResourcePostgresCatalog) New(client *databricks.WorkspaceClient) *ResourcePostgresCatalog { + return &ResourcePostgresCatalog{client: client} +} + +func (*ResourcePostgresCatalog) PrepareState(input *resources.PostgresCatalog) *PostgresCatalogState { + return &PostgresCatalogState{ + CatalogId: input.CatalogId, + CatalogCatalogSpec: input.CatalogCatalogSpec, + } +} + +func (*ResourcePostgresCatalog) RemapState(remote *PostgresCatalogRemote) *PostgresCatalogState { + return &PostgresCatalogState{ + CatalogId: remote.CatalogId, + CatalogCatalogSpec: remote.CatalogCatalogSpec, + } +} + +// makePostgresCatalogRemote converts the SDK Catalog into the embedded remote shape. +// GET does not echo spec today (only status is returned); the embedded spec fields +// stay at their zero values, and resources.yml suppresses phantom drift via +// ignore_remote_changes with reason spec:input_only. +// +// Status.CatalogId is the short identifier and matches the user-supplied config. +// Prefer it over parsing remote.Name — semantic contract from the API rather than +// string manipulation on the hierarchical path. +func makePostgresCatalogRemote(catalog *postgres.Catalog) *PostgresCatalogRemote { + var spec postgres.CatalogCatalogSpec + if catalog.Spec != nil { + spec = *catalog.Spec + } + var catalogId string + if catalog.Status != nil { + catalogId = catalog.Status.CatalogId + } + return &PostgresCatalogRemote{ + CatalogCatalogSpec: spec, + CatalogId: catalogId, + Name: catalog.Name, + Status: catalog.Status, + Uid: catalog.Uid, + CreateTime: catalog.CreateTime, + UpdateTime: catalog.UpdateTime, + } +} + +func (r *ResourcePostgresCatalog) DoRead(ctx context.Context, id string) (*PostgresCatalogRemote, error) { + catalog, err := r.client.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: id}) + if err != nil { + return nil, err + } + return makePostgresCatalogRemote(catalog), nil +} + +func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *PostgresCatalogState) (string, *PostgresCatalogRemote, error) { + waiter, err := r.client.Postgres.CreateCatalog(ctx, postgres.CreateCatalogRequest{ + CatalogId: config.CatalogId, + Catalog: postgres.Catalog{ + Spec: &config.CatalogCatalogSpec, + + // Output-only fields. + CreateTime: nil, + Name: "", + Status: nil, + Uid: "", + UpdateTime: nil, + ForceSendFields: nil, + }, + }) + if err != nil { + return "", nil, err + } + + result, err := waiter.Wait(ctx) + if err != nil { + return "", nil, err + } + remote := makePostgresCatalogRemote(result) + return remote.Name, remote, nil +} + +func (r *ResourcePostgresCatalog) DoDelete(ctx context.Context, id string) error { + waiter, err := r.client.Postgres.DeleteCatalog(ctx, postgres.DeleteCatalogRequest{ + Name: id, + }) + if err != nil { + return err + } + return waiter.Wait(ctx) +} diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index 85c15d6f343..ee9a4892d9c 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -208,6 +208,20 @@ resources: - field: ttl reason: spec:input_only + postgres_catalogs: + + recreate_on_changes: + - field: postgres_database + reason: spec:immutable + + ignore_remote_changes: + - field: branch + reason: spec:input_only + - field: create_database_if_missing + reason: spec:input_only + - field: postgres_database + reason: spec:input_only + postgres_endpoints: recreate_on_changes: diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 81aec6e74d1..8c8a6ce07c5 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -512,6 +512,20 @@ resources: - field: replace_existing reason: "input_only; cannot be updated after create" + postgres_catalogs: + recreate_on_changes: + # catalog_id is part of the hierarchical name and immutable. + - field: catalog_id + reason: immutable + # The Postgres SDK has no UpdateCatalog endpoint, so any local change + # requires delete+create. The OpenAPI spec only marks postgres_database + # as IMMUTABLE (handled by autogen); branch and create_database_if_missing + # need explicit entries here. + - field: branch + reason: immutable + - field: create_database_if_missing + reason: immutable + vector_search_endpoints: recreate_on_changes: - field: endpoint_type diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f15766a0d3b..0f2e1b0c798 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -215,6 +215,9 @@ github.com/databricks/cli/bundle/config.Resources: "postgres_branches": "description": |- PLACEHOLDER + "postgres_catalogs": + "description": |- + The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. "postgres_endpoints": "description": |- PLACEHOLDER @@ -796,6 +799,22 @@ github.com/databricks/cli/bundle/config/resources.PostgresBranch: "update_time": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.PostgresCatalog: + "branch": + "description": |- + PLACEHOLDER + "catalog_id": + "description": |- + PLACEHOLDER + "create_database_if_missing": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "postgres_database": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.PostgresEndpoint: "autoscaling_limit_max_cu": "description": |- diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index db86398accb..ad2edcd333b 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -219,6 +219,8 @@ var RequiredFields = map[string][]string{ "resources.postgres_branches.*": {"branch_id", "parent"}, + "resources.postgres_catalogs.*": {"postgres_database", "catalog_id"}, + "resources.postgres_endpoints.*": {"endpoint_type", "endpoint_id", "parent"}, "resources.postgres_endpoints.*.group": {"max", "min"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index ac57cedbe53..389efd115bf 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1412,6 +1412,39 @@ } ] }, + "resources.PostgresCatalog": { + "oneOf": [ + { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "catalog_id": { + "$ref": "#/$defs/string" + }, + "create_database_if_missing": { + "$ref": "#/$defs/bool" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "postgres_database": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_id", + "postgres_database" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresEndpoint": { "oneOf": [ { @@ -2528,6 +2561,10 @@ "postgres_branches": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresBranch" }, + "postgres_catalogs": { + "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + }, "postgres_endpoints": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresEndpoint" }, @@ -11750,6 +11787,20 @@ } ] }, + "resources.PostgresCatalog": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresEndpoint": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index 0748cf84e47..ab06243d227 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -1391,6 +1391,31 @@ "parent" ] }, + "resources.PostgresCatalog": { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "catalog_id": { + "$ref": "#/$defs/string" + }, + "create_database_if_missing": { + "$ref": "#/$defs/bool" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "postgres_database": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_id", + "postgres_database" + ] + }, "resources.PostgresEndpoint": { "type": "object", "properties": { @@ -2486,6 +2511,10 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresBranch", "x-since-version": "v0.287.0" }, + "postgres_catalogs": { + "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + }, "postgres_endpoints": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresEndpoint", "x-since-version": "v0.287.0" @@ -9716,6 +9745,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresBranch" } }, + "resources.PostgresCatalog": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + } + }, "resources.PostgresEndpoint": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 34c4fa4f5aa..097b708e62d 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -49,6 +49,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, + "resources.postgres_catalogs.test_postgres_catalog": {ID: "catalogs/test_catalog"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, } err := StateToBundle(t.Context(), state, &config) @@ -118,6 +119,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "projects/test-project/branches/main/endpoints/primary", config.Resources.PostgresEndpoints["test_postgres_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresEndpoints["test_postgres_endpoint"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -292,6 +296,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "test_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -374,6 +385,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.PostgresEndpoints["test_postgres_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresEndpoints["test_postgres_endpoint"].ModifiedStatus) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -661,6 +675,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "test_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + "test_postgres_catalog_new": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog_new", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -716,6 +742,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.postgres_branches.test_postgres_branch_old": {ID: "projects/test-project/branches/old"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, "resources.postgres_endpoints.test_postgres_endpoint_old": {ID: "projects/test-project/branches/main/endpoints/old"}, + "resources.postgres_catalogs.test_postgres_catalog": {ID: "catalogs/test_catalog"}, + "resources.postgres_catalogs.test_postgres_catalog_old": {ID: "catalogs/test_catalog_old"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, } @@ -864,6 +892,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.PostgresEndpoints["test_postgres_endpoint_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresEndpoints["test_postgres_endpoint_new"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog_old", config.Resources.PostgresCatalogs["test_postgres_catalog_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresCatalogs["test_postgres_catalog_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresCatalogs["test_postgres_catalog_new"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) assert.Equal(t, "vs-endpoint-old", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_old"].ID) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 6fc686c4fd3..d00a5039b8c 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -171,6 +171,7 @@ type FakeWorkspace struct { PostgresProjects map[string]postgres.Project PostgresBranches map[string]postgres.Branch PostgresEndpoints map[string]postgres.Endpoint + PostgresCatalogs map[string]postgres.Catalog PostgresOperations map[string]postgres.Operation // Branches and endpoints that the server provisioned implicitly together @@ -306,6 +307,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PostgresProjects: map[string]postgres.Project{}, PostgresBranches: map[string]postgres.Branch{}, PostgresEndpoints: map[string]postgres.Endpoint{}, + PostgresCatalogs: map[string]postgres.Catalog{}, PostgresOperations: map[string]postgres.Operation{}, postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8a1188a79ec..c78fd2b3ad1 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -938,6 +938,27 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresEndpointDelete(name) }) + // Postgres Catalogs: + server.Handle("POST", "/api/2.0/postgres/catalogs", func(req Request) any { + catalogID := req.URL.Query().Get("catalog_id") + return req.Workspace.PostgresCatalogCreate(req, catalogID) + }) + + server.Handle("GET", "/api/2.0/postgres/catalogs/{id}", func(req Request) any { + return req.Workspace.PostgresCatalogGet("catalogs/" + req.Vars["id"]) + }) + + server.Handle("DELETE", "/api/2.0/postgres/catalogs/{id}", func(req Request) any { + return req.Workspace.PostgresCatalogDelete("catalogs/" + req.Vars["id"]) + }) + + // Operations for catalogs are nested under the resource. Matches the real + // API and what the SDK polls based on the operation.Name we return. + server.Handle("GET", "/api/2.0/postgres/catalogs/{id}/operations/{operation_id}", func(req Request) any { + name := "catalogs/" + req.Vars["id"] + "/operations/" + req.Vars["operation_id"] + return req.Workspace.PostgresOperationGet(name) + }) + // Catch-all handler for invalid postgres resource names. // This handles cases like GET /api/2.0/postgres/1234 where "1234" is not a valid resource name. server.Handle("GET", "/api/2.0/postgres/{name}", func(req Request) any { diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 8a7c22b4ecf..e720f00701e 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -685,6 +685,83 @@ func (s *FakeWorkspace) PostgresEndpointDelete(name string) Response { } } +// PostgresCatalogCreate creates a new postgres catalog. +func (s *FakeWorkspace) PostgresCatalogCreate(req Request, catalogID string) Response { + defer s.LockUnlock()() + + if catalogID == "" { + return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", `Field 'catalog_id' is required, expected non-default value (not "")!`) + } + + // The SDK sends request.Catalog (the inner struct) as the body — NOT a + // {"catalog": ...} wrapper. Unmarshal directly into postgres.Catalog. + var catalog postgres.Catalog + if len(req.Body) > 0 { + if err := json.Unmarshal(req.Body, &catalog); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + } + } + } + + name := "catalogs/" + catalogID + + if _, exists := s.PostgresCatalogs[name]; exists { + return postgresErrorResponse(409, "ALREADY_EXISTS", "catalog with such id already exists") + } + + now := nowTime() + catalog.Name = name + catalog.Uid = nextUUID() + catalog.CreateTime = now + catalog.UpdateTime = now + + status := &postgres.CatalogCatalogStatus{ + CatalogId: catalogID, + } + if catalog.Spec != nil { + status.Branch = catalog.Spec.Branch + status.PostgresDatabase = catalog.Spec.PostgresDatabase + // Project portion of the status is "projects/{project_id}", derived + // from the branch name "projects/{project_id}/branches/{branch_id}". + if idx := strings.Index(catalog.Spec.Branch, "/branches/"); idx > 0 { + status.Project = catalog.Spec.Branch[:idx] + } + } + catalog.Status = status + + // Real API only returns status on GET (no spec). Match that to keep + // acceptance test output stable between local and cloud. + catalog.Spec = nil + + s.PostgresCatalogs[name] = catalog + + return Response{Body: s.createOperationLocked(name, catalog)} +} + +// PostgresCatalogGet retrieves a postgres catalog by name. +func (s *FakeWorkspace) PostgresCatalogGet(name string) Response { + defer s.LockUnlock()() + + catalog, exists := s.PostgresCatalogs[name] + if !exists { + return postgresNotFoundResponse("catalog") + } + return Response{Body: catalog} +} + +// PostgresCatalogDelete deletes a postgres catalog. +func (s *FakeWorkspace) PostgresCatalogDelete(name string) Response { + defer s.LockUnlock()() + + if _, exists := s.PostgresCatalogs[name]; !exists { + return postgresNotFoundResponse("catalog") + } + delete(s.PostgresCatalogs, name) + return Response{Body: s.createOperationLocked(name, nil)} +} + // PostgresOperationGet retrieves a postgres operation by name. func (s *FakeWorkspace) PostgresOperationGet(name string) Response { defer s.LockUnlock()() @@ -706,9 +783,12 @@ func (s *FakeWorkspace) createOperationLocked(resourceName string, response any) // Determine resource type from name for metadata @type resourceType := "Project" - if strings.Contains(resourceName, "/endpoints/") { + switch { + case strings.HasPrefix(resourceName, "catalogs/"): + resourceType = "Catalog" + case strings.Contains(resourceName, "/endpoints/"): resourceType = "Endpoint" - } else if strings.Contains(resourceName, "/branches/") { + case strings.Contains(resourceName, "/branches/"): resourceType = "Branch" }