From 3857f0f0a026fd9ec66683593114f84c58702131 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 20:55:15 +0200 Subject: [PATCH 1/6] Add postgres_catalogs bundle resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Changes New `postgres_catalogs` resource binding a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. Supported on both direct and terraform deployment engines. The spec fields are classified as both `recreate_on_changes` and `ignore_remote_changes: input_only`. The two cover orthogonal diffs the planner runs — recreate fires on local edits to an immutable field, and ignore_remote silences the phantom drift from GET not echoing spec back today. Lift the `input_only` entries once the backend starts returning spec. ## Tests Acceptance coverage: `basic` and `recreate` exercise each engine, plus the existing `no_drift` and `migrate` invariants pick up the new resource. Both engines produce identical human-readable output and identical wire bodies; only the captured request streams diverge by filename (`out.requests.{direct,terraform}.json`). Verified end to end on a live workspace: the bundle deploys a project and catalog, a row written directly into the bound Postgres database becomes visible through the UC federated view, and a follow-up write shows up on re-read. This pull request and its description were written by Isaac. --- .agent/skills/pr-checklist/SKILL.md | 3 +- NEXT_CHANGELOG.md | 1 + .../configs/postgres_catalog.yml.tmpl | 15 +++ .../invariant/continue_293/out.test.toml | 1 + .../bundle/invariant/migrate/out.test.toml | 1 + .../bundle/invariant/no_drift/out.test.toml | 1 + acceptance/bundle/invariant/test.toml | 2 + acceptance/bundle/refschema/out.fields.txt | 22 +++ .../basic/databricks.yml.tmpl | 19 +++ .../basic/out.requests.direct.json | 31 +++++ .../basic/out.requests.terraform.json | 31 +++++ .../postgres_catalogs/basic/out.test.toml | 6 + .../postgres_catalogs/basic/output.txt | 74 +++++++++++ .../resources/postgres_catalogs/basic/script | 23 ++++ .../recreate/databricks.yml.tmpl | 19 +++ .../postgres_catalogs/recreate/out.test.toml | 6 + .../postgres_catalogs/recreate/output.txt | 31 +++++ .../postgres_catalogs/recreate/script | 16 +++ .../resources/postgres_catalogs/test.toml | 26 ++++ .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 8 ++ .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/resources.go | 3 + bundle/config/resources/postgres_catalog.go | 66 +++++++++ bundle/config/resources_test.go | 8 ++ bundle/deploy/terraform/interpolate.go | 2 +- bundle/deploy/terraform/pkg.go | 1 + .../tfdyn/convert_postgres_catalog.go | 56 ++++++++ .../tfdyn/convert_postgres_catalog_test.go | 73 ++++++++++ bundle/deploy/terraform/util.go | 2 +- bundle/direct/dresources/all.go | 1 + bundle/direct/dresources/all_test.go | 6 + .../direct/dresources/apitypes.generated.yml | 2 + bundle/direct/dresources/apitypes.yml | 2 + bundle/direct/dresources/postgres_catalog.go | 86 ++++++++++++ .../direct/dresources/resources.generated.yml | 14 ++ bundle/direct/dresources/resources.yml | 31 +++++ bundle/direct/dresources/type_test.go | 6 + bundle/internal/schema/annotations.yml | 19 +++ .../validation/generated/required_fields.go | 2 + bundle/schema/jsonschema.json | 51 +++++++ bundle/schema/jsonschema_for_docs.json | 125 ++++++++++++++---- bundle/statemgmt/state_load_test.go | 35 +++++ libs/testserver/fake_workspace.go | 2 + libs/testserver/handlers.go | 21 +++ libs/testserver/postgres.go | 84 +++++++++++- 46 files changed, 1003 insertions(+), 34 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/output.txt create mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/script create mode 100644 acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_catalogs/recreate/output.txt create mode 100644 acceptance/bundle/resources/postgres_catalogs/recreate/script create mode 100644 acceptance/bundle/resources/postgres_catalogs/test.toml create mode 100644 bundle/config/resources/postgres_catalog.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go create mode 100644 bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go create mode 100644 bundle/direct/dresources/postgres_catalog.go diff --git a/.agent/skills/pr-checklist/SKILL.md b/.agent/skills/pr-checklist/SKILL.md index 86559354fcb..e01058a0120 100644 --- a/.agent/skills/pr-checklist/SKILL.md +++ b/.agent/skills/pr-checklist/SKILL.md @@ -16,8 +16,9 @@ Before submitting a PR, run these commands to match what CI checks. CI uses the # 3. Tests (CI runs with both deployment engines) ./task test -# 4. If you changed bundle config structs or schema-related code: +# 4. If you changed bundle config structs, schema, or direct-engine resource code: ./task generate-schema +./task generate-direct # 5. If you changed files in python/: ./task pydabs-codegen pydabs-test pydabs-lint pydabs-docs diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index fa2adeb1e8a..a92ec7b7914 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,5 +10,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) +* Add `postgres_catalogs` resource to bind a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. ### Dependency updates diff --git a/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl b/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl new file mode 100644 index 00000000000..2ae10852504 --- /dev/null +++ b/acceptance/bundle/invariant/configs/postgres_catalog.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + postgres_projects: + project: + project_id: test-pg-project-$UNIQUE_NAME + display_name: Test Postgres Project + + postgres_catalogs: + catalog: + catalog_id: test_pg_catalog_$UNIQUE_NAME + branch: ${resources.postgres_projects.project.name}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 11aaf584918..3bff3e2836d 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -26,6 +26,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 257e33005a3..abfb10c5f73 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -44,6 +44,7 @@ EnvMatrix.INPUT_CONFIG = [ "pipeline.yml.tmpl", "pipeline_config_dots.yml.tmpl", "postgres_branch.yml.tmpl", + "postgres_catalog.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", @@ -67,6 +68,7 @@ no_alert_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=alert.yml.tmpl"] no_postgres_project_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_project.yml.tmpl"] no_postgres_branch_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_branch.yml.tmpl"] no_postgres_endpoint_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_endpoint.yml.tmpl"] +no_postgres_catalog_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=postgres_catalog.yml.tmpl"] # External locations require actual storage credentials with cloud IAM setup # which are environment-specific, so we only test locally with the mock server diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 999296b2139..2c3ad788ea3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2733,6 +2733,28 @@ resources.postgres_branches.*.ttl *duration.Duration INPUT STATE resources.postgres_branches.*.uid string REMOTE resources.postgres_branches.*.update_time *time.Time REMOTE resources.postgres_branches.*.url string INPUT +resources.postgres_catalogs.*.branch string INPUT STATE +resources.postgres_catalogs.*.catalog_id string INPUT STATE +resources.postgres_catalogs.*.create_database_if_missing bool INPUT STATE +resources.postgres_catalogs.*.create_time *time.Time REMOTE +resources.postgres_catalogs.*.id string INPUT +resources.postgres_catalogs.*.lifecycle resources.Lifecycle INPUT +resources.postgres_catalogs.*.lifecycle.prevent_destroy bool INPUT +resources.postgres_catalogs.*.modified_status string INPUT +resources.postgres_catalogs.*.name string REMOTE +resources.postgres_catalogs.*.postgres_database string INPUT STATE +resources.postgres_catalogs.*.spec *postgres.CatalogCatalogSpec REMOTE +resources.postgres_catalogs.*.spec.branch string REMOTE +resources.postgres_catalogs.*.spec.create_database_if_missing bool REMOTE +resources.postgres_catalogs.*.spec.postgres_database string REMOTE +resources.postgres_catalogs.*.status *postgres.CatalogCatalogStatus REMOTE +resources.postgres_catalogs.*.status.branch string REMOTE +resources.postgres_catalogs.*.status.catalog_id string REMOTE +resources.postgres_catalogs.*.status.postgres_database string REMOTE +resources.postgres_catalogs.*.status.project string REMOTE +resources.postgres_catalogs.*.uid string REMOTE +resources.postgres_catalogs.*.update_time *time.Time REMOTE +resources.postgres_catalogs.*.url string INPUT resources.postgres_endpoints.*.autoscaling_limit_max_cu float64 INPUT STATE resources.postgres_endpoints.*.autoscaling_limit_min_cu float64 INPUT STATE resources.postgres_endpoints.*.create_time *time.Time REMOTE diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..b339f4c1354 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: deploy-postgres-catalog-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Catalog" + pg_version: 16 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json new file mode 100644 index 00000000000..891733f4150 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json @@ -0,0 +1,31 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Catalog", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "lakebase_test_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/catalogs/lakebase_test_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json new file mode 100644 index 00000000000..891733f4150 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json @@ -0,0 +1,31 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Test Project for Catalog", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/catalogs", + "q": { + "catalog_id": "lakebase_test_[UNIQUE_NAME]" + }, + "body": { + "spec": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "create_database_if_missing": true, + "postgres_database": "appdb" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/catalogs/lakebase_test_[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/output.txt b/acceptance/bundle/resources/postgres_catalogs/basic/output.txt new file mode 100644 index 00000000000..7bf45891cb3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/output.txt @@ -0,0 +1,74 @@ + +>>> [CLI] bundle validate +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default +Resources: + Postgres catalogs: + my_catalog: + Name: lakebase_test_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME] + Postgres projects: + my_project: + Name: Test Project for Catalog + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] postgres get-catalog catalogs/lakebase_test_[UNIQUE_NAME] +{ + "name": "catalogs/lakebase_test_[UNIQUE_NAME]", + "status": { + "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "catalog_id": "lakebase_test_[UNIQUE_NAME]", + "postgres_database": "appdb", + "project": "projects/test-pg-proj-[UNIQUE_NAME]" + } +} + +>>> [CLI] bundle summary +Name: deploy-postgres-catalog-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default +Resources: + Postgres catalogs: + my_catalog: + Name: lakebase_test_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/lakebase_test_[UNIQUE_NAME] + Postgres projects: + my_project: + Name: Test Project for Catalog + URL: (not deployed) + +>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-catalog-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/script b/acceptance/bundle/resources/postgres_catalogs/basic/script new file mode 100644 index 00000000000..ecf81072cfb --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/basic/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +trace $CLI bundle summary + +rm -f out.requests.txt +trace $CLI bundle deploy + +# Get catalog details. Hide volatile fields so cloud and local match. +catalog_name="catalogs/lakebase_test_${UNIQUE_NAME}" +trace $CLI postgres get-catalog "${catalog_name}" | jq 'del(.create_time, .update_time, .uid)' + +trace $CLI bundle summary + +# Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..bf3c2b56576 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: recreate-postgres-catalog-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Catalog Recreate" + pg_version: 16 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: $POSTGRES_DATABASE + create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt b/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt new file mode 100644 index 00000000000..1f6bb621987 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/output.txt @@ -0,0 +1,31 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +recreate postgres_catalogs.my_catalog + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-postgres-catalog-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/script b/acceptance/bundle/resources/postgres_catalogs/recreate/script new file mode 100644 index 00000000000..4d9de784299 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +export POSTGRES_DATABASE=appdb +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy + +# Toggle a recreate-on-change field; plan must show delete + create. +export POSTGRES_DATABASE=otherdb +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle plan | contains.py "Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged" + +trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_catalogs/test.toml b/acceptance/bundle/resources/postgres_catalogs/test.toml new file mode 100644 index 00000000000..66f0811b343 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/test.toml @@ -0,0 +1,26 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +# Lakebase v2 (postgres) is only available in AWS as of January 2026 +CloudEnvs.gcp = false +CloudEnvs.azure = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Clean up ?o= suffix after URL since not all workspaces have that +Old = '\?o=\[(NUMID|ALPHANUMID)\]' +New = '' +Order = 1000 + +[[Repls]] +# Normalize postgres operation IDs (unique per operation). +Old = '/operations/[A-Za-z0-9+/=-]+' +New = '/operations/[OPERATION_ID]' +Order = 2000 diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index e472241f282..d78c1ad8145 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -28,6 +28,7 @@ var unsupportedResources = []string{ "synced_database_tables", "postgres_branches", "postgres_endpoints", + "postgres_catalogs", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index fe9c9a1db06..927c7a19132 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -247,6 +247,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "postgres_catalog1": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "postgres_catalog_1", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "vs_endpoint1": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -441,6 +448,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { "PostgresProjects", "PostgresBranches", "PostgresEndpoints", + "PostgresCatalogs", } diags := bundle.ApplySeq(t.Context(), b, ApplyTargetMode(), ApplyPresets()) diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 0b7003f5873..58b27113a52 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -46,6 +46,7 @@ func allResourceTypes(t *testing.T) []string { "models", "pipelines", "postgres_branches", + "postgres_catalogs", "postgres_endpoints", "postgres_projects", "quality_monitors", @@ -174,6 +175,7 @@ var allowList = []string{ "pipelines", "models", "postgres_branches", + "postgres_catalogs", "postgres_endpoints", "postgres_projects", "registered_models", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 225ec32165d..30a7f4e95e5 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -35,6 +35,7 @@ type Resources struct { PostgresProjects map[string]*resources.PostgresProject `json:"postgres_projects,omitempty"` PostgresBranches map[string]*resources.PostgresBranch `json:"postgres_branches,omitempty"` PostgresEndpoints map[string]*resources.PostgresEndpoint `json:"postgres_endpoints,omitempty"` + PostgresCatalogs map[string]*resources.PostgresCatalog `json:"postgres_catalogs,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` } @@ -112,6 +113,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_projects"], r.PostgresProjects), collectResourceMap(descriptions["postgres_branches"], r.PostgresBranches), collectResourceMap(descriptions["postgres_endpoints"], r.PostgresEndpoints), + collectResourceMap(descriptions["postgres_catalogs"], r.PostgresCatalogs), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), } } @@ -167,6 +169,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), + "postgres_catalogs": (&resources.PostgresCatalog{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), } } diff --git a/bundle/config/resources/postgres_catalog.go b/bundle/config/resources/postgres_catalog.go new file mode 100644 index 00000000000..c17ffaa1682 --- /dev/null +++ b/bundle/config/resources/postgres_catalog.go @@ -0,0 +1,66 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type PostgresCatalogConfig struct { + postgres.CatalogCatalogSpec + + // CatalogId is the user-specified UC catalog name. Becomes the trailing + // component of the server-assigned Name: "catalogs/{catalog_id}". + CatalogId string `json:"catalog_id"` +} + +func (c *PostgresCatalogConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c *PostgresCatalogConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type PostgresCatalog struct { + BaseResource + PostgresCatalogConfig +} + +func (c *PostgresCatalog) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: name}) + if err != nil { + log.Debugf(ctx, "postgres catalog %s does not exist", name) + return false, err + } + return true, nil +} + +func (c *PostgresCatalog) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "postgres_catalog", + PluralName: "postgres_catalogs", + SingularTitle: "Postgres catalog", + PluralTitle: "Postgres catalogs", + } +} + +func (c *PostgresCatalog) GetName() string { + return c.CatalogId +} + +func (c *PostgresCatalog) GetURL() string { + return c.URL +} + +func (c *PostgresCatalog) InitializeURL(baseURL url.URL) { + if c.CatalogId == "" { + return + } + baseURL.Path = "explore/data/" + c.CatalogId + c.URL = baseURL.String() +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 943b279a288..626fac07983 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -273,6 +273,13 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "my_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "my_postgres_catalog", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -312,6 +319,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetProject(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetBranch(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockPostgresAPI().EXPECT().GetCatalog(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) allResources := supportedResources.AllResources() diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index fdcb671bdd3..7a1acb3d4e6 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -16,7 +16,7 @@ type interpolateMutator struct{} // Postgres resources use "name" instead of "id" as their identifier attribute. func isPostgresResource(resourceType string) bool { switch resourceType { - case "postgres_projects", "postgres_branches", "postgres_endpoints": + case "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs": return true default: return false diff --git a/bundle/deploy/terraform/pkg.go b/bundle/deploy/terraform/pkg.go index a66e5cb6a06..ff68c1b1513 100644 --- a/bundle/deploy/terraform/pkg.go +++ b/bundle/deploy/terraform/pkg.go @@ -129,6 +129,7 @@ var GroupToTerraformName = map[string]string{ "postgres_projects": "databricks_postgres_project", "postgres_branches": "databricks_postgres_branch", "postgres_endpoints": "databricks_postgres_endpoint", + "postgres_catalogs": "databricks_postgres_catalog", // 3 level groups: resources.*.GROUP "permissions": "databricks_permissions", diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go new file mode 100644 index 00000000000..b84fb09d717 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog.go @@ -0,0 +1,56 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" +) + +type postgresCatalogConverter struct{} + +func (c postgresCatalogConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + // The bundle config has flattened CatalogCatalogSpec fields at the top level. + // Terraform expects them nested in a "spec" block. + specFields := specFieldNames(schema.ResourcePostgresCatalogSpec{}) + topLevelFields := []string{"catalog_id"} + + specMap := make(map[string]dyn.Value) + for _, field := range specFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + specMap[field] = v + } + } + + outMap := make(map[string]dyn.Value) + for _, field := range topLevelFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + outMap[field] = v + } + } + if len(specMap) > 0 { + outMap["spec"] = dyn.V(specMap) + } + + vout := dyn.V(outMap) + + vout, diags := convert.Normalize(schema.ResourcePostgresCatalog{}, vout) + for _, diag := range diags { + log.Debugf(ctx, "postgres catalog normalization diagnostic: %s", diag.Summary) + } + + vout, err := convertLifecycle(ctx, vout, vin.Get("lifecycle")) + if err != nil { + return err + } + + out.PostgresCatalog[key] = vout.AsAny() + + return nil +} + +func init() { + registerConverter("postgres_catalogs", postgresCatalogConverter{}) +} diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go new file mode 100644 index 00000000000..8eca766f613 --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_catalog_test.go @@ -0,0 +1,73 @@ +package tfdyn + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertPostgresCatalog(t *testing.T) { + src := resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "shop_lakebase", + CatalogCatalogSpec: postgres.CatalogCatalogSpec{ + Branch: "projects/my-shop/branches/production", + PostgresDatabase: "appdb", + CreateDatabaseIfMissing: true, + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresCatalogConverter{}.Convert(ctx, "my_postgres_catalog", vin, out) + require.NoError(t, err) + + postgresCatalog := out.PostgresCatalog["my_postgres_catalog"] + assert.Equal(t, map[string]any{ + "catalog_id": "shop_lakebase", + "spec": map[string]any{ + "branch": "projects/my-shop/branches/production", + "postgres_database": "appdb", + "create_database_if_missing": true, + }, + }, postgresCatalog) +} + +func TestConvertPostgresCatalogMinimal(t *testing.T) { + src := resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "minimal_catalog", + CatalogCatalogSpec: postgres.CatalogCatalogSpec{ + Branch: "projects/my-shop/branches/production", + PostgresDatabase: "appdb", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresCatalogConverter{}.Convert(ctx, "minimal_postgres_catalog", vin, out) + require.NoError(t, err) + + postgresCatalog := out.PostgresCatalog["minimal_postgres_catalog"] + assert.Equal(t, map[string]any{ + "catalog_id": "minimal_catalog", + "spec": map[string]any{ + "branch": "projects/my-shop/branches/production", + "postgres_database": "appdb", + }, + }, postgresCatalog) +} diff --git a/bundle/deploy/terraform/util.go b/bundle/deploy/terraform/util.go index 632d32bca19..69c3c4f886d 100644 --- a/bundle/deploy/terraform/util.go +++ b/bundle/deploy/terraform/util.go @@ -96,7 +96,7 @@ func parseResourcesState(ctx context.Context, path string) (ExportedResourcesMap // The direct engine manages permissions as a sub-resource // (SecretScopeFixups adds MANAGE ACL for the current user). result[resourceKey+".permissions"] = ResourceState{ID: instance.Attributes.Name} - case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints": + case "apps", "database_instances", "database_catalogs", "synced_database_tables", "postgres_projects", "postgres_branches", "postgres_endpoints", "postgres_catalogs": resourceKey = "resources." + groupName + "." + resource.Name resourceState = ResourceState{ID: instance.Attributes.Name} case "dashboards": diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index ddc30c41f54..497bf9bcb7a 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -23,6 +23,7 @@ var SupportedResources = map[string]any{ "postgres_projects": (*ResourcePostgresProject)(nil), "postgres_branches": (*ResourcePostgresBranch)(nil), "postgres_endpoints": (*ResourcePostgresEndpoint)(nil), + "postgres_catalogs": (*ResourcePostgresCatalog)(nil), "alerts": (*ResourceAlert)(nil), "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 2c0a2e52f22..f130afc6194 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -191,6 +191,12 @@ var testConfig map[string]any = map[string]any{ }, }, + "postgres_catalogs": &resources.PostgresCatalog{ + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + "alerts": &resources.Alert{ AlertV2: sql.AlertV2{ DisplayName: "my-alert", diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index a80b3baa69b..6478f029feb 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -28,6 +28,8 @@ pipelines: pipelines.CreatePipeline postgres_branches: postgres.BranchSpec +postgres_catalogs: postgres.CatalogCatalogSpec + postgres_endpoints: postgres.EndpointSpec postgres_projects: postgres.ProjectStatus diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index 29db9b67b20..c37dfbccbb1 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -9,3 +9,5 @@ postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec postgres_projects: postgres.ProjectSpec + +postgres_catalogs: postgres.CatalogCatalogSpec diff --git a/bundle/direct/dresources/postgres_catalog.go b/bundle/direct/dresources/postgres_catalog.go new file mode 100644 index 00000000000..497e52e9fb8 --- /dev/null +++ b/bundle/direct/dresources/postgres_catalog.go @@ -0,0 +1,86 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/postgres" +) + +type ResourcePostgresCatalog struct { + client *databricks.WorkspaceClient +} + +type PostgresCatalogState = resources.PostgresCatalogConfig + +func (*ResourcePostgresCatalog) New(client *databricks.WorkspaceClient) *ResourcePostgresCatalog { + return &ResourcePostgresCatalog{client: client} +} + +func (*ResourcePostgresCatalog) PrepareState(input *resources.PostgresCatalog) *PostgresCatalogState { + return &PostgresCatalogState{ + CatalogId: input.CatalogId, + CatalogCatalogSpec: input.CatalogCatalogSpec, + } +} + +func (*ResourcePostgresCatalog) RemapState(remote *postgres.Catalog) *PostgresCatalogState { + // The server-side ID is the full hierarchical name `catalogs/`. + // Keep it as-is — the `catalogs/` prefix is an inherent part of the ID, not + // noise to strip. + // + // GET does not return the spec today (only status). Return an empty spec + // and rely on the ignore_remote_changes / input_only classifications in + // resources.yml to suppress phantom drift until the backend starts + // echoing spec values on GET. + return &PostgresCatalogState{ + CatalogId: remote.Name, + CatalogCatalogSpec: postgres.CatalogCatalogSpec{ + Branch: "", + CreateDatabaseIfMissing: false, + PostgresDatabase: "", + ForceSendFields: nil, + }, + } +} + +func (r *ResourcePostgresCatalog) DoRead(ctx context.Context, id string) (*postgres.Catalog, error) { + return r.client.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: id}) +} + +func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *PostgresCatalogState) (string, *postgres.Catalog, error) { + waiter, err := r.client.Postgres.CreateCatalog(ctx, postgres.CreateCatalogRequest{ + CatalogId: config.CatalogId, + Catalog: postgres.Catalog{ + Spec: &config.CatalogCatalogSpec, + + // Output-only fields. + CreateTime: nil, + Name: "", + Status: nil, + Uid: "", + UpdateTime: nil, + ForceSendFields: nil, + }, + }) + if err != nil { + return "", nil, err + } + + result, err := waiter.Wait(ctx) + if err != nil { + return "", nil, err + } + return result.Name, result, nil +} + +func (r *ResourcePostgresCatalog) DoDelete(ctx context.Context, id string) error { + waiter, err := r.client.Postgres.DeleteCatalog(ctx, postgres.DeleteCatalogRequest{ + Name: id, + }) + if err != nil { + return err + } + return waiter.Wait(ctx) +} diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index 85c15d6f343..ee9a4892d9c 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -208,6 +208,20 @@ resources: - field: ttl reason: spec:input_only + postgres_catalogs: + + recreate_on_changes: + - field: postgres_database + reason: spec:immutable + + ignore_remote_changes: + - field: branch + reason: spec:input_only + - field: create_database_if_missing + reason: spec:input_only + - field: postgres_database + reason: spec:input_only + postgres_endpoints: recreate_on_changes: diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 569fca9ee82..3deb4066742 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -508,6 +508,37 @@ resources: - field: endpoint_id reason: immutable + postgres_catalogs: + recreate_on_changes: + # catalog_id is part of the hierarchical name and immutable. + - field: catalog_id + reason: immutable + # The Postgres SDK has no UpdateCatalog endpoint, so any local change to + # a spec field requires delete+create. recreate_on_changes fires on the + # intent diff (local config vs. persisted state); ignore_remote_changes + # below silences the orthogonal drift diff (persisted state vs. RemapState). + - field: branch + reason: immutable + - field: postgres_database + reason: immutable + - field: create_database_if_missing + reason: immutable + ignore_remote_changes: + # The remote .Name always carries the "catalogs/" prefix while the + # user-supplied catalog_id does not. The framework's state ID still + # holds the full prefixed name; this just suppresses the cosmetic diff. + - field: catalog_id + reason: input_only + # The catalog API does not currently echo back spec values on GET; the + # missing remote values would otherwise trigger phantom drift. Lift + # these once the backend starts returning spec on GET. + - field: branch + reason: input_only + - field: postgres_database + reason: input_only + - field: create_database_if_missing + reason: input_only + vector_search_endpoints: recreate_on_changes: - field: endpoint_type diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 88f246723bf..6fd68a596e0 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -75,6 +75,12 @@ var knownMissingInRemoteType = map[string][]string{ "pg_version", "project_id", }, + "postgres_catalogs": { + "branch", + "catalog_id", + "create_database_if_missing", + "postgres_database", + }, "vector_search_endpoints": { "target_qps", "usage_policy_id", diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f6ac5c45d4d..68d796cd091 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -215,6 +215,9 @@ github.com/databricks/cli/bundle/config.Resources: "postgres_branches": "description": |- PLACEHOLDER + "postgres_catalogs": + "description": |- + The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. "postgres_endpoints": "description": |- PLACEHOLDER @@ -793,6 +796,22 @@ github.com/databricks/cli/bundle/config/resources.PostgresBranch: "update_time": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.PostgresCatalog: + "branch": + "description": |- + PLACEHOLDER + "catalog_id": + "description": |- + PLACEHOLDER + "create_database_if_missing": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "postgres_database": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.PostgresEndpoint: "autoscaling_limit_max_cu": "description": |- diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index db86398accb..ad2edcd333b 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -219,6 +219,8 @@ var RequiredFields = map[string][]string{ "resources.postgres_branches.*": {"branch_id", "parent"}, + "resources.postgres_catalogs.*": {"postgres_database", "catalog_id"}, + "resources.postgres_endpoints.*": {"endpoint_type", "endpoint_id", "parent"}, "resources.postgres_endpoints.*.group": {"max", "min"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 26e773f64a4..c05cd06f610 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1409,6 +1409,39 @@ } ] }, + "resources.PostgresCatalog": { + "oneOf": [ + { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "catalog_id": { + "$ref": "#/$defs/string" + }, + "create_database_if_missing": { + "$ref": "#/$defs/bool" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "postgres_database": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_id", + "postgres_database" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresEndpoint": { "oneOf": [ { @@ -2522,6 +2555,10 @@ "postgres_branches": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresBranch" }, + "postgres_catalogs": { + "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + }, "postgres_endpoints": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresEndpoint" }, @@ -11744,6 +11781,20 @@ } ] }, + "resources.PostgresCatalog": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.PostgresEndpoint": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index 0748cf84e47..ff64ac25c26 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -1391,6 +1391,31 @@ "parent" ] }, + "resources.PostgresCatalog": { + "type": "object", + "properties": { + "branch": { + "$ref": "#/$defs/string" + }, + "catalog_id": { + "$ref": "#/$defs/string" + }, + "create_database_if_missing": { + "$ref": "#/$defs/bool" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "postgres_database": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_id", + "postgres_database" + ] + }, "resources.PostgresEndpoint": { "type": "object", "properties": { @@ -1942,7 +1967,8 @@ "x-since-version": "v0.298.0" }, "target_qps": { - "$ref": "#/$defs/int64" + "$ref": "#/$defs/int64", + "x-since-version": "v0.299.2" }, "usage_policy_id": { "$ref": "#/$defs/string", @@ -2486,6 +2512,10 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresBranch", "x-since-version": "v0.287.0" }, + "postgres_catalogs": { + "description": "The Postgres catalog definitions for the bundle, where each key is the name of the catalog. Each entry binds a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + }, "postgres_endpoints": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.PostgresEndpoint", "x-since-version": "v0.287.0" @@ -4286,7 +4316,8 @@ "description": "The confidential computing technology for this cluster's instances.\nCurrently only SEV_SNP is supported, and only on N2D instance types.\nWhen not set, no confidential computing is applied.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ConfidentialComputeType", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" }, "first_on_demand": { "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", @@ -6759,7 +6790,8 @@ "properties": { "include_confluence_spaces": { "description": "(Optional) Spaces to filter Confluence data on", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -6783,7 +6815,8 @@ "properties": { "confluence_options": { "description": "Confluence specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ConfluenceConnectorOptions" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ConfluenceConnectorOptions", + "x-since-version": "v0.299.2" }, "gdrive_options": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.GoogleDriveOptions", @@ -6800,17 +6833,20 @@ }, "jira_options": { "description": "Jira specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions", + "x-since-version": "v0.299.2" }, "meta_ads_options": { "description": "Meta Marketing (Meta Ads) specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions", + "x-since-version": "v0.299.2" }, "outlook_options": { "description": "Outlook specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" }, "sharepoint_options": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SharepointOptions", @@ -6822,7 +6858,8 @@ "description": "Smartsheet specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SmartsheetOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" }, "tiktok_ads_options": { "description": "TikTok Ads specific options for ingestion", @@ -6835,7 +6872,8 @@ "description": "Zendesk Support specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ZendeskSupportOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -7063,7 +7101,8 @@ "properties": { "manager_account_id": { "description": "(Required) Manager Account ID (also called MCC Account ID) used to list and access\ncustomer accounts under this manager account. This is required for fetching the list\nof customer accounts during source selection.\nIf the same field is also set in the object-level GoogleAdsOptions (connector_options),\nthe object-level value takes precedence over this top-level config.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -7422,7 +7461,8 @@ "properties": { "include_jira_spaces": { "description": "(Optional) Projects to filter Jira data on", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -7437,35 +7477,43 @@ "properties": { "action_attribution_windows": { "description": "(Optional) Action attribution windows for insights reporting (e.g. \"28d_click\", \"1d_view\")", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "action_breakdowns": { "description": "(Optional) Action breakdowns to configure for data aggregation", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "action_report_time": { "description": "(Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime)", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" }, "breakdowns": { "description": "(Optional) Breakdowns to configure for data aggregation", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "custom_insights_lookback_window": { "description": "(Optional) Window in days to revisit data during sync to capture\nupdated conversion data from the API.", - "$ref": "#/$defs/int" + "$ref": "#/$defs/int", + "x-since-version": "v0.299.2" }, "level": { "description": "(Optional) Granularity of data to pull (account, ad, adset, campaign)", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" }, "start_date": { "description": "(Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added\nafter this date will be ingested", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" }, "time_increment": { "description": "(Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days)", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -7546,48 +7594,58 @@ "properties": { "attachment_mode": { "description": "(Optional) Controls which attachments to ingest.\nIf not specified, defaults to ALL.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookAttachmentMode" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookAttachmentMode", + "x-since-version": "v0.299.2" }, "body_format": { "description": "(Optional) Defines how the body_content column is populated.\nTEXT_HTML: Preserves full formatting, links, and styling.\nTEXT_PLAIN: Converts body to plain text. Recommended for AI/RAG pipelines to reduce token usage and noise.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookBodyFormat" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookBodyFormat", + "x-since-version": "v0.299.2" }, "folder_filter": { "description": "Deprecated. Use include_folders instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", + "x-since-version": "v0.299.2", "deprecated": true }, "include_folders": { "description": "(Optional) Filter mail folders to include in the sync.\nIf not specified, all folders will be synced.\nExamples: Inbox, Sent Items, Custom_Folder\nFilter semantics: OR between different folders.", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "include_mailboxes": { "description": "(Optional) List of mailboxes to sync (e.g. mailbox email addresses or identifiers).\nIf not specified, all accessible mailboxes are ingested.\nFilter semantics: OR between different mailboxes.", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "include_senders": { "description": "(Optional) Filter emails by sender address. Uses exact email match.\nExamples: user@vendor.com, alerts@system.io, noreply@company.com\nIf not specified, emails from all senders will be synced.\nFilter semantics: OR between different senders.", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "include_subjects": { "description": "(Optional) Filter emails by subject line. Values ending with \"*\" use prefix match (subject starts with\nthe part before \"*\"); otherwise substring match (subject contains the value).\nExamples: \"Invoice\" (substring), \"Re:*\" (prefix), \"Support Ticket\", \"URGENT*\"\nIf not specified, emails with all subjects will be synced.\nFilter semantics: OR between different subjects.", - "$ref": "#/$defs/slice/string" + "$ref": "#/$defs/slice/string", + "x-since-version": "v0.299.2" }, "sender_filter": { "description": "Deprecated. Use include_senders instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", + "x-since-version": "v0.299.2", "deprecated": true }, "start_date": { "description": "(Optional) Start date for the initial sync in YYYY-MM-DD format.\nFormat: YYYY-MM-DD (e.g., 2024-01-01)\nThis determines the earliest date from which to sync historical data.\nIf not specified, complete history is ingested.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" }, "subject_filter": { "description": "Deprecated. Use include_subjects instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", + "x-since-version": "v0.299.2", "deprecated": true } }, @@ -8025,7 +8083,8 @@ "properties": { "enforce_schema": { "description": "(Optional) When true, maps each column to its Smartsheet-declared type (Text/Number/Date/\nCheckbox/etc.). Cells that do not conform to the declared type are set to NULL.\nWhen false, all columns land as STRING. Use false for sheets with irregular data or columns\nthat frequently violate their own declared type.\nIf not specified, defaults to true.", - "$ref": "#/$defs/bool" + "$ref": "#/$defs/bool", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -8058,7 +8117,8 @@ "google_ads_config": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.GoogleAdsConfig", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -8252,7 +8312,8 @@ "properties": { "start_date": { "description": "(Optional) Start date in YYYY-MM-DD format for the initial sync.\nThis determines the earliest date from which to sync historical data.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.299.2" } }, "additionalProperties": false @@ -9716,6 +9777,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresBranch" } }, + "resources.PostgresCatalog": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.PostgresCatalog" + } + }, "resources.PostgresEndpoint": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 34c4fa4f5aa..097b708e62d 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -49,6 +49,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, + "resources.postgres_catalogs.test_postgres_catalog": {ID: "catalogs/test_catalog"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, } err := StateToBundle(t.Context(), state, &config) @@ -118,6 +119,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "projects/test-project/branches/main/endpoints/primary", config.Resources.PostgresEndpoints["test_postgres_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresEndpoints["test_postgres_endpoint"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -292,6 +296,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "test_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -374,6 +385,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.PostgresEndpoints["test_postgres_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresEndpoints["test_postgres_endpoint"].ModifiedStatus) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) @@ -661,6 +675,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + PostgresCatalogs: map[string]*resources.PostgresCatalog{ + "test_postgres_catalog": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog", + }, + }, + "test_postgres_catalog_new": { + PostgresCatalogConfig: resources.PostgresCatalogConfig{ + CatalogId: "test_catalog_new", + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "test_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -716,6 +742,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.postgres_branches.test_postgres_branch_old": {ID: "projects/test-project/branches/old"}, "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, "resources.postgres_endpoints.test_postgres_endpoint_old": {ID: "projects/test-project/branches/main/endpoints/old"}, + "resources.postgres_catalogs.test_postgres_catalog": {ID: "catalogs/test_catalog"}, + "resources.postgres_catalogs.test_postgres_catalog_old": {ID: "catalogs/test_catalog_old"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, } @@ -864,6 +892,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.PostgresEndpoints["test_postgres_endpoint_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresEndpoints["test_postgres_endpoint_new"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog", config.Resources.PostgresCatalogs["test_postgres_catalog"].ID) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog"].ModifiedStatus) + assert.Equal(t, "catalogs/test_catalog_old", config.Resources.PostgresCatalogs["test_postgres_catalog_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresCatalogs["test_postgres_catalog_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.PostgresCatalogs["test_postgres_catalog_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresCatalogs["test_postgres_catalog_new"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) assert.Equal(t, "vs-endpoint-old", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_old"].ID) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 5430c68cbcc..34e5d2a2106 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -171,6 +171,7 @@ type FakeWorkspace struct { PostgresProjects map[string]postgres.Project PostgresBranches map[string]postgres.Branch PostgresEndpoints map[string]postgres.Endpoint + PostgresCatalogs map[string]postgres.Catalog PostgresOperations map[string]postgres.Operation // clusterVenvs caches Python venvs per existing cluster ID, @@ -299,6 +300,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PostgresProjects: map[string]postgres.Project{}, PostgresBranches: map[string]postgres.Branch{}, PostgresEndpoints: map[string]postgres.Endpoint{}, + PostgresCatalogs: map[string]postgres.Catalog{}, PostgresOperations: map[string]postgres.Operation{}, clusterVenvs: map[string]*clusterEnv{}, Alerts: map[string]sql.AlertV2{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index d98011fc7ba..d9dd4203686 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -936,6 +936,27 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.PostgresEndpointDelete(name) }) + // Postgres Catalogs: + server.Handle("POST", "/api/2.0/postgres/catalogs", func(req Request) any { + catalogID := req.URL.Query().Get("catalog_id") + return req.Workspace.PostgresCatalogCreate(req, catalogID) + }) + + server.Handle("GET", "/api/2.0/postgres/catalogs/{id}", func(req Request) any { + return req.Workspace.PostgresCatalogGet("catalogs/" + req.Vars["id"]) + }) + + server.Handle("DELETE", "/api/2.0/postgres/catalogs/{id}", func(req Request) any { + return req.Workspace.PostgresCatalogDelete("catalogs/" + req.Vars["id"]) + }) + + // Operations for catalogs are nested under the resource. Matches the real + // API and what the SDK polls based on the operation.Name we return. + server.Handle("GET", "/api/2.0/postgres/catalogs/{id}/operations/{operation_id}", func(req Request) any { + name := "catalogs/" + req.Vars["id"] + "/operations/" + req.Vars["operation_id"] + return req.Workspace.PostgresOperationGet(name) + }) + // Catch-all handler for invalid postgres resource names. // This handles cases like GET /api/2.0/postgres/1234 where "1234" is not a valid resource name. server.Handle("GET", "/api/2.0/postgres/{name}", func(req Request) any { diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index f3a488b5704..eae426eb432 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -587,6 +587,83 @@ func (s *FakeWorkspace) PostgresEndpointDelete(name string) Response { } } +// PostgresCatalogCreate creates a new postgres catalog. +func (s *FakeWorkspace) PostgresCatalogCreate(req Request, catalogID string) Response { + defer s.LockUnlock()() + + if catalogID == "" { + return postgresErrorResponse(400, "INVALID_PARAMETER_VALUE", `Field 'catalog_id' is required, expected non-default value (not "")!`) + } + + // The SDK sends request.Catalog (the inner struct) as the body — NOT a + // {"catalog": ...} wrapper. Unmarshal directly into postgres.Catalog. + var catalog postgres.Catalog + if len(req.Body) > 0 { + if err := json.Unmarshal(req.Body, &catalog); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + } + } + } + + name := "catalogs/" + catalogID + + if _, exists := s.PostgresCatalogs[name]; exists { + return postgresErrorResponse(409, "ALREADY_EXISTS", "catalog with such id already exists") + } + + now := nowTime() + catalog.Name = name + catalog.Uid = nextUUID() + catalog.CreateTime = now + catalog.UpdateTime = now + + status := &postgres.CatalogCatalogStatus{ + CatalogId: catalogID, + } + if catalog.Spec != nil { + status.Branch = catalog.Spec.Branch + status.PostgresDatabase = catalog.Spec.PostgresDatabase + // Project portion of the status is "projects/{project_id}", derived + // from the branch name "projects/{project_id}/branches/{branch_id}". + if idx := strings.Index(catalog.Spec.Branch, "/branches/"); idx > 0 { + status.Project = catalog.Spec.Branch[:idx] + } + } + catalog.Status = status + + // Real API only returns status on GET (no spec). Match that to keep + // acceptance test output stable between local and cloud. + catalog.Spec = nil + + s.PostgresCatalogs[name] = catalog + + return Response{Body: s.createOperationLocked(name, catalog)} +} + +// PostgresCatalogGet retrieves a postgres catalog by name. +func (s *FakeWorkspace) PostgresCatalogGet(name string) Response { + defer s.LockUnlock()() + + catalog, exists := s.PostgresCatalogs[name] + if !exists { + return postgresNotFoundResponse("catalog") + } + return Response{Body: catalog} +} + +// PostgresCatalogDelete deletes a postgres catalog. +func (s *FakeWorkspace) PostgresCatalogDelete(name string) Response { + defer s.LockUnlock()() + + if _, exists := s.PostgresCatalogs[name]; !exists { + return postgresNotFoundResponse("catalog") + } + delete(s.PostgresCatalogs, name) + return Response{Body: s.createOperationLocked(name, nil)} +} + // PostgresOperationGet retrieves a postgres operation by name. func (s *FakeWorkspace) PostgresOperationGet(name string) Response { defer s.LockUnlock()() @@ -608,9 +685,12 @@ func (s *FakeWorkspace) createOperationLocked(resourceName string, response any) // Determine resource type from name for metadata @type resourceType := "Project" - if strings.Contains(resourceName, "/endpoints/") { + switch { + case strings.HasPrefix(resourceName, "catalogs/"): + resourceType = "Catalog" + case strings.Contains(resourceName, "/endpoints/"): resourceType = "Endpoint" - } else if strings.Contains(resourceName, "/branches/") { + case strings.Contains(resourceName, "/branches/"): resourceType = "Branch" } From bbbf49fd35acd6808a84267b9876e7c15de04453 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 18 May 2026 22:36:42 +0200 Subject: [PATCH 2/6] changelog: link postgres_catalogs entry to PR This pull request and its description were written by Isaac. --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index a92ec7b7914..3c77bfe3128 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,6 +10,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) -* Add `postgres_catalogs` resource to bind a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch. +* Add `postgres_catalogs` resource to bind a Unity Catalog catalog to a Postgres database on a Lakebase Autoscaling branch ([#5265](https://github.com/databricks/cli/pull/5265)). ### Dependency updates From 647bd802d4ecd8ce4805cf48944bbcfab07046a4 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 12:01:30 +0200 Subject: [PATCH 3/6] Drop redundant postgres_catalogs lifecycle rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hand-written `branch`, `postgres_database`, and `create_database_if_missing` entries under `ignore_remote_changes` are already produced by the OpenAPI autogen (`spec:input_only`); `postgres_database` is also autogen'd under `recreate_on_changes` (`spec:immutable`). Drop the duplicates from `resources.yml`. `catalog_id` was in `ignore_remote_changes` only to mask the cosmetic `catalogs/` prefix that the old `RemapState` propagated from `remote.Name`. Source it from `remote.Status.CatalogId` instead — semantic contract from the API rather than string manipulation on the hierarchical path — and drop the entry. `catalog_id` stays in `recreate_on_changes` (synthetic hierarchical key, not in the API spec) along with `branch` and `create_database_if_missing` (no UpdateCatalog endpoint). Co-authored-by: Isaac --- bundle/direct/dresources/postgres_catalog.go | 16 ++++++++----- bundle/direct/dresources/resources.yml | 25 ++++---------------- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/bundle/direct/dresources/postgres_catalog.go b/bundle/direct/dresources/postgres_catalog.go index 497e52e9fb8..9da3becbc32 100644 --- a/bundle/direct/dresources/postgres_catalog.go +++ b/bundle/direct/dresources/postgres_catalog.go @@ -26,16 +26,20 @@ func (*ResourcePostgresCatalog) PrepareState(input *resources.PostgresCatalog) * } func (*ResourcePostgresCatalog) RemapState(remote *postgres.Catalog) *PostgresCatalogState { - // The server-side ID is the full hierarchical name `catalogs/`. - // Keep it as-is — the `catalogs/` prefix is an inherent part of the ID, not - // noise to strip. + // Status.CatalogId is the short identifier and matches the user-supplied + // config. Prefer it over parsing remote.Name — semantic contract from the + // API rather than string manipulation on the hierarchical path. // // GET does not return the spec today (only status). Return an empty spec - // and rely on the ignore_remote_changes / input_only classifications in - // resources.yml to suppress phantom drift until the backend starts + // and rely on the spec:input_only classifications generated from the + // OpenAPI schema to suppress phantom drift until the backend starts // echoing spec values on GET. + var catalogId string + if remote.Status != nil { + catalogId = remote.Status.CatalogId + } return &PostgresCatalogState{ - CatalogId: remote.Name, + CatalogId: catalogId, CatalogCatalogSpec: postgres.CatalogCatalogSpec{ Branch: "", CreateDatabaseIfMissing: false, diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 3deb4066742..1269b881a94 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -513,31 +513,14 @@ resources: # catalog_id is part of the hierarchical name and immutable. - field: catalog_id reason: immutable - # The Postgres SDK has no UpdateCatalog endpoint, so any local change to - # a spec field requires delete+create. recreate_on_changes fires on the - # intent diff (local config vs. persisted state); ignore_remote_changes - # below silences the orthogonal drift diff (persisted state vs. RemapState). + # The Postgres SDK has no UpdateCatalog endpoint, so any local change + # requires delete+create. The OpenAPI spec only marks postgres_database + # as IMMUTABLE (handled by autogen); branch and create_database_if_missing + # need explicit entries here. - field: branch reason: immutable - - field: postgres_database - reason: immutable - field: create_database_if_missing reason: immutable - ignore_remote_changes: - # The remote .Name always carries the "catalogs/" prefix while the - # user-supplied catalog_id does not. The framework's state ID still - # holds the full prefixed name; this just suppresses the cosmetic diff. - - field: catalog_id - reason: input_only - # The catalog API does not currently echo back spec values on GET; the - # missing remote values would otherwise trigger phantom drift. Lift - # these once the backend starts returning spec on GET. - - field: branch - reason: input_only - - field: postgres_database - reason: input_only - - field: create_database_if_missing - reason: input_only vector_search_endpoints: recreate_on_changes: From 3ce34ddb7b87034de072d0225b3face2ca26827f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 15:30:11 +0200 Subject: [PATCH 4/6] acc: consolidate postgres_catalogs/basic request files Direct and terraform engines produced identical output. Per the repo rule in .agent/rules/testing.md, only diverging files should be split into per-engine variants; this matches the precedent set by postgres_projects/basic. Co-authored-by: Isaac --- ...requests.direct.json => out.requests.json} | 0 .../basic/out.requests.terraform.json | 31 ------------------- .../resources/postgres_catalogs/basic/script | 2 +- 3 files changed, 1 insertion(+), 32 deletions(-) rename acceptance/bundle/resources/postgres_catalogs/basic/{out.requests.direct.json => out.requests.json} (100%) delete mode 100644 acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.json similarity index 100% rename from acceptance/bundle/resources/postgres_catalogs/basic/out.requests.direct.json rename to acceptance/bundle/resources/postgres_catalogs/basic/out.requests.json diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json b/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json deleted file mode 100644 index 891733f4150..00000000000 --- a/acceptance/bundle/resources/postgres_catalogs/basic/out.requests.terraform.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "method": "POST", - "path": "/api/2.0/postgres/projects", - "q": { - "project_id": "test-pg-proj-[UNIQUE_NAME]" - }, - "body": { - "spec": { - "display_name": "Test Project for Catalog", - "pg_version": 16 - } - } -} -{ - "method": "POST", - "path": "/api/2.0/postgres/catalogs", - "q": { - "catalog_id": "lakebase_test_[UNIQUE_NAME]" - }, - "body": { - "spec": { - "branch": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", - "create_database_if_missing": true, - "postgres_database": "appdb" - } - } -} -{ - "method": "GET", - "path": "/api/2.0/postgres/catalogs/lakebase_test_[UNIQUE_NAME]" -} diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/script b/acceptance/bundle/resources/postgres_catalogs/basic/script index ecf81072cfb..ba6240838c8 100644 --- a/acceptance/bundle/resources/postgres_catalogs/basic/script +++ b/acceptance/bundle/resources/postgres_catalogs/basic/script @@ -20,4 +20,4 @@ trace $CLI postgres get-catalog "${catalog_name}" | jq 'del(.create_time, .updat trace $CLI bundle summary # Filter requests to only show postgres operations (exclude workspace, telemetry, and operation polling). -trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.json From 0ecbfabe56299e1d2c2147e24ae40feba429aba2 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 16:37:03 +0200 Subject: [PATCH 5/6] postgres: embed spec on Catalog remote type Apply the same hoist that #5273 does for Branch/Endpoint/Project. Define PostgresCatalogRemote that embeds CatalogCatalogSpec and exposes the identifier and output-only fields at the top level. DoRead returns the new shape so state-side and remote-side paths line up, which is a prerequisite for drift detection on spec fields once the backend echoes spec on GET. Today the embedded fields are auto-classified spec:input_only from the API field behaviors in resources.generated.yml, so drift is correctly suppressed. Prompted by https://github.com/databricks/cli/pull/5265#discussion_r3266274249. Should land after #5273. Co-authored-by: Isaac --- acceptance/bundle/refschema/out.fields.txt | 12 +-- bundle/direct/dresources/postgres_catalog.go | 89 +++++++++++++++----- bundle/direct/dresources/type_test.go | 6 -- 3 files changed, 70 insertions(+), 37 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 2c3ad788ea3..7b585ae01c7 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2733,20 +2733,16 @@ resources.postgres_branches.*.ttl *duration.Duration INPUT STATE resources.postgres_branches.*.uid string REMOTE resources.postgres_branches.*.update_time *time.Time REMOTE resources.postgres_branches.*.url string INPUT -resources.postgres_catalogs.*.branch string INPUT STATE -resources.postgres_catalogs.*.catalog_id string INPUT STATE -resources.postgres_catalogs.*.create_database_if_missing bool INPUT STATE +resources.postgres_catalogs.*.branch string ALL +resources.postgres_catalogs.*.catalog_id string ALL +resources.postgres_catalogs.*.create_database_if_missing bool ALL resources.postgres_catalogs.*.create_time *time.Time REMOTE resources.postgres_catalogs.*.id string INPUT resources.postgres_catalogs.*.lifecycle resources.Lifecycle INPUT resources.postgres_catalogs.*.lifecycle.prevent_destroy bool INPUT resources.postgres_catalogs.*.modified_status string INPUT resources.postgres_catalogs.*.name string REMOTE -resources.postgres_catalogs.*.postgres_database string INPUT STATE -resources.postgres_catalogs.*.spec *postgres.CatalogCatalogSpec REMOTE -resources.postgres_catalogs.*.spec.branch string REMOTE -resources.postgres_catalogs.*.spec.create_database_if_missing bool REMOTE -resources.postgres_catalogs.*.spec.postgres_database string REMOTE +resources.postgres_catalogs.*.postgres_database string ALL resources.postgres_catalogs.*.status *postgres.CatalogCatalogStatus REMOTE resources.postgres_catalogs.*.status.branch string REMOTE resources.postgres_catalogs.*.status.catalog_id string REMOTE diff --git a/bundle/direct/dresources/postgres_catalog.go b/bundle/direct/dresources/postgres_catalog.go index 9da3becbc32..48a3262e4ad 100644 --- a/bundle/direct/dresources/postgres_catalog.go +++ b/bundle/direct/dresources/postgres_catalog.go @@ -5,9 +5,36 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/postgres" ) +// PostgresCatalogRemote is the return type for DoRead. It embeds CatalogCatalogSpec so +// that all paths in StateType are valid paths in RemoteType, enabling drift detection +// for spec fields once the backend echoes spec on GET. +type PostgresCatalogRemote struct { + postgres.CatalogCatalogSpec + + CatalogId string `json:"catalog_id,omitempty"` + + Name string `json:"name,omitempty"` + Status *postgres.CatalogCatalogStatus `json:"status,omitempty"` + Uid string `json:"uid,omitempty"` + CreateTime *sdktime.Time `json:"create_time,omitempty"` + UpdateTime *sdktime.Time `json:"update_time,omitempty"` +} + +// Custom marshaler needed because embedded CatalogCatalogSpec has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *PostgresCatalogRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s PostgresCatalogRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourcePostgresCatalog struct { client *databricks.WorkspaceClient } @@ -25,35 +52,50 @@ func (*ResourcePostgresCatalog) PrepareState(input *resources.PostgresCatalog) * } } -func (*ResourcePostgresCatalog) RemapState(remote *postgres.Catalog) *PostgresCatalogState { - // Status.CatalogId is the short identifier and matches the user-supplied - // config. Prefer it over parsing remote.Name — semantic contract from the - // API rather than string manipulation on the hierarchical path. - // - // GET does not return the spec today (only status). Return an empty spec - // and rely on the spec:input_only classifications generated from the - // OpenAPI schema to suppress phantom drift until the backend starts - // echoing spec values on GET. +func (*ResourcePostgresCatalog) RemapState(remote *PostgresCatalogRemote) *PostgresCatalogState { + return &PostgresCatalogState{ + CatalogId: remote.CatalogId, + CatalogCatalogSpec: remote.CatalogCatalogSpec, + } +} + +// makePostgresCatalogRemote converts the SDK Catalog into the embedded remote shape. +// GET does not echo spec today (only status is returned); the embedded spec fields +// stay at their zero values, and resources.yml suppresses phantom drift via +// ignore_remote_changes with reason spec:input_only. +// +// Status.CatalogId is the short identifier and matches the user-supplied config. +// Prefer it over parsing remote.Name — semantic contract from the API rather than +// string manipulation on the hierarchical path. +func makePostgresCatalogRemote(catalog *postgres.Catalog) *PostgresCatalogRemote { + var spec postgres.CatalogCatalogSpec + if catalog.Spec != nil { + spec = *catalog.Spec + } var catalogId string - if remote.Status != nil { - catalogId = remote.Status.CatalogId + if catalog.Status != nil { + catalogId = catalog.Status.CatalogId } - return &PostgresCatalogState{ - CatalogId: catalogId, - CatalogCatalogSpec: postgres.CatalogCatalogSpec{ - Branch: "", - CreateDatabaseIfMissing: false, - PostgresDatabase: "", - ForceSendFields: nil, - }, + return &PostgresCatalogRemote{ + CatalogCatalogSpec: spec, + CatalogId: catalogId, + Name: catalog.Name, + Status: catalog.Status, + Uid: catalog.Uid, + CreateTime: catalog.CreateTime, + UpdateTime: catalog.UpdateTime, } } -func (r *ResourcePostgresCatalog) DoRead(ctx context.Context, id string) (*postgres.Catalog, error) { - return r.client.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: id}) +func (r *ResourcePostgresCatalog) DoRead(ctx context.Context, id string) (*PostgresCatalogRemote, error) { + catalog, err := r.client.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: id}) + if err != nil { + return nil, err + } + return makePostgresCatalogRemote(catalog), nil } -func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *PostgresCatalogState) (string, *postgres.Catalog, error) { +func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *PostgresCatalogState) (string, *PostgresCatalogRemote, error) { waiter, err := r.client.Postgres.CreateCatalog(ctx, postgres.CreateCatalogRequest{ CatalogId: config.CatalogId, Catalog: postgres.Catalog{ @@ -76,7 +118,8 @@ func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *Postgres if err != nil { return "", nil, err } - return result.Name, result, nil + remote := makePostgresCatalogRemote(result) + return remote.Name, remote, nil } func (r *ResourcePostgresCatalog) DoDelete(ctx context.Context, id string) error { diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 6fd68a596e0..88f246723bf 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -75,12 +75,6 @@ var knownMissingInRemoteType = map[string][]string{ "pg_version", "project_id", }, - "postgres_catalogs": { - "branch", - "catalog_id", - "create_database_if_missing", - "postgres_database", - }, "vector_search_endpoints": { "target_qps", "usage_policy_id", From 37c80702f9f8bcabbcee0c7a208032c4460fd3db Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 19 May 2026 16:55:03 +0200 Subject: [PATCH 6/6] docs schema: drop unrelated x-since-version v0.299.2 annotations These annotations crept into the schema regeneration when the postgres_catalogs commit (3857f0f0a) was first generated; they belong to other fields and should not ride along with this PR. Keep only the postgres_catalogs additions. Co-authored-by: Isaac --- bundle/schema/jsonschema_for_docs.json | 90 +++++++++----------------- 1 file changed, 29 insertions(+), 61 deletions(-) diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index ff64ac25c26..ab06243d227 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -1967,8 +1967,7 @@ "x-since-version": "v0.298.0" }, "target_qps": { - "$ref": "#/$defs/int64", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/int64" }, "usage_policy_id": { "$ref": "#/$defs/string", @@ -4316,8 +4315,7 @@ "description": "The confidential computing technology for this cluster's instances.\nCurrently only SEV_SNP is supported, and only on N2D instance types.\nWhen not set, no confidential computing is applied.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/compute.ConfidentialComputeType", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true, - "x-since-version": "v0.299.2" + "doNotSuggest": true }, "first_on_demand": { "description": "The first `first_on_demand` nodes of the cluster will be placed on on-demand instances.\nThis value should be greater than 0, to make sure the cluster driver node is placed on an\non-demand instance. If this value is greater than or equal to the current cluster size, all\nnodes will be placed on on-demand instances. If this value is less than the current cluster\nsize, `first_on_demand` nodes will be placed on on-demand instances and the remainder will\nbe placed on `availability` instances. Note that this value does not affect\ncluster size and cannot currently be mutated over the lifetime of a cluster.", @@ -6790,8 +6788,7 @@ "properties": { "include_confluence_spaces": { "description": "(Optional) Spaces to filter Confluence data on", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" } }, "additionalProperties": false @@ -6815,8 +6812,7 @@ "properties": { "confluence_options": { "description": "Confluence specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ConfluenceConnectorOptions", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ConfluenceConnectorOptions" }, "gdrive_options": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.GoogleDriveOptions", @@ -6833,20 +6829,17 @@ }, "jira_options": { "description": "Jira specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.JiraConnectorOptions" }, "meta_ads_options": { "description": "Meta Marketing (Meta Ads) specific options for ingestion", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.MetaMarketingOptions" }, "outlook_options": { "description": "Outlook specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true, - "x-since-version": "v0.299.2" + "doNotSuggest": true }, "sharepoint_options": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SharepointOptions", @@ -6858,8 +6851,7 @@ "description": "Smartsheet specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.SmartsheetOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true, - "x-since-version": "v0.299.2" + "doNotSuggest": true }, "tiktok_ads_options": { "description": "TikTok Ads specific options for ingestion", @@ -6872,8 +6864,7 @@ "description": "Zendesk Support specific options for ingestion", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ZendeskSupportOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true, - "x-since-version": "v0.299.2" + "doNotSuggest": true } }, "additionalProperties": false @@ -7101,8 +7092,7 @@ "properties": { "manager_account_id": { "description": "(Required) Manager Account ID (also called MCC Account ID) used to list and access\ncustomer accounts under this manager account. This is required for fetching the list\nof customer accounts during source selection.\nIf the same field is also set in the object-level GoogleAdsOptions (connector_options),\nthe object-level value takes precedence over this top-level config.", - "$ref": "#/$defs/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/string" } }, "additionalProperties": false @@ -7461,8 +7451,7 @@ "properties": { "include_jira_spaces": { "description": "(Optional) Projects to filter Jira data on", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" } }, "additionalProperties": false @@ -7477,43 +7466,35 @@ "properties": { "action_attribution_windows": { "description": "(Optional) Action attribution windows for insights reporting (e.g. \"28d_click\", \"1d_view\")", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" }, "action_breakdowns": { "description": "(Optional) Action breakdowns to configure for data aggregation", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" }, "action_report_time": { "description": "(Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime)", - "$ref": "#/$defs/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/string" }, "breakdowns": { "description": "(Optional) Breakdowns to configure for data aggregation", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" }, "custom_insights_lookback_window": { "description": "(Optional) Window in days to revisit data during sync to capture\nupdated conversion data from the API.", - "$ref": "#/$defs/int", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/int" }, "level": { "description": "(Optional) Granularity of data to pull (account, ad, adset, campaign)", - "$ref": "#/$defs/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/string" }, "start_date": { "description": "(Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added\nafter this date will be ingested", - "$ref": "#/$defs/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/string" }, "time_increment": { "description": "(Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days)", - "$ref": "#/$defs/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/string" } }, "additionalProperties": false @@ -7594,58 +7575,48 @@ "properties": { "attachment_mode": { "description": "(Optional) Controls which attachments to ingest.\nIf not specified, defaults to ALL.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookAttachmentMode", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookAttachmentMode" }, "body_format": { "description": "(Optional) Defines how the body_content column is populated.\nTEXT_HTML: Preserves full formatting, links, and styling.\nTEXT_PLAIN: Converts body to plain text. Recommended for AI/RAG pipelines to reduce token usage and noise.", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookBodyFormat", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.OutlookBodyFormat" }, "folder_filter": { "description": "Deprecated. Use include_folders instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", - "x-since-version": "v0.299.2", "deprecated": true }, "include_folders": { "description": "(Optional) Filter mail folders to include in the sync.\nIf not specified, all folders will be synced.\nExamples: Inbox, Sent Items, Custom_Folder\nFilter semantics: OR between different folders.", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" }, "include_mailboxes": { "description": "(Optional) List of mailboxes to sync (e.g. mailbox email addresses or identifiers).\nIf not specified, all accessible mailboxes are ingested.\nFilter semantics: OR between different mailboxes.", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" }, "include_senders": { "description": "(Optional) Filter emails by sender address. Uses exact email match.\nExamples: user@vendor.com, alerts@system.io, noreply@company.com\nIf not specified, emails from all senders will be synced.\nFilter semantics: OR between different senders.", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" }, "include_subjects": { "description": "(Optional) Filter emails by subject line. Values ending with \"*\" use prefix match (subject starts with\nthe part before \"*\"); otherwise substring match (subject contains the value).\nExamples: \"Invoice\" (substring), \"Re:*\" (prefix), \"Support Ticket\", \"URGENT*\"\nIf not specified, emails with all subjects will be synced.\nFilter semantics: OR between different subjects.", - "$ref": "#/$defs/slice/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/slice/string" }, "sender_filter": { "description": "Deprecated. Use include_senders instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", - "x-since-version": "v0.299.2", "deprecated": true }, "start_date": { "description": "(Optional) Start date for the initial sync in YYYY-MM-DD format.\nFormat: YYYY-MM-DD (e.g., 2024-01-01)\nThis determines the earliest date from which to sync historical data.\nIf not specified, complete history is ingested.", - "$ref": "#/$defs/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/string" }, "subject_filter": { "description": "Deprecated. Use include_subjects instead.", "$ref": "#/$defs/slice/string", "deprecationMessage": "This field is deprecated", - "x-since-version": "v0.299.2", "deprecated": true } }, @@ -8083,8 +8054,7 @@ "properties": { "enforce_schema": { "description": "(Optional) When true, maps each column to its Smartsheet-declared type (Text/Number/Date/\nCheckbox/etc.). Cells that do not conform to the declared type are set to NULL.\nWhen false, all columns land as STRING. Use false for sheets with irregular data or columns\nthat frequently violate their own declared type.\nIf not specified, defaults to true.", - "$ref": "#/$defs/bool", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/bool" } }, "additionalProperties": false @@ -8117,8 +8087,7 @@ "google_ads_config": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.GoogleAdsConfig", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true, - "x-since-version": "v0.299.2" + "doNotSuggest": true } }, "additionalProperties": false @@ -8312,8 +8281,7 @@ "properties": { "start_date": { "description": "(Optional) Start date in YYYY-MM-DD format for the initial sync.\nThis determines the earliest date from which to sync historical data.", - "$ref": "#/$defs/string", - "x-since-version": "v0.299.2" + "$ref": "#/$defs/string" } }, "additionalProperties": false