diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 36cbb445d2c..2fd7b638a63 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -8,5 +8,6 @@ ### Bundles * Make sure warnings asking for approval are understood by agents ([#5239](https://github.com/databricks/cli/pull/5239)) +* Support `replace_existing: true` on `postgres_branches` and `postgres_endpoints` so bundles can manage the implicitly-created production branch and primary read-write endpoint of a Lakebase project. ### Dependency updates diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 999296b2139..decf27ca913 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2706,6 +2706,7 @@ resources.postgres_branches.*.modified_status string INPUT resources.postgres_branches.*.name string REMOTE resources.postgres_branches.*.no_expiry bool INPUT STATE resources.postgres_branches.*.parent string ALL +resources.postgres_branches.*.replace_existing bool INPUT STATE resources.postgres_branches.*.source_branch string INPUT STATE resources.postgres_branches.*.source_branch_lsn string INPUT STATE resources.postgres_branches.*.source_branch_time *time.Time INPUT STATE @@ -2750,6 +2751,7 @@ resources.postgres_endpoints.*.modified_status string INPUT resources.postgres_endpoints.*.name string REMOTE resources.postgres_endpoints.*.no_suspension bool INPUT STATE resources.postgres_endpoints.*.parent string ALL +resources.postgres_endpoints.*.replace_existing bool INPUT STATE resources.postgres_endpoints.*.settings *postgres.EndpointSettings INPUT STATE resources.postgres_endpoints.*.settings.pg_settings map[string]string INPUT STATE resources.postgres_endpoints.*.settings.pg_settings.* string INPUT STATE diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_branches/replace_existing/databricks.yml.tmpl new file mode 100644 index 00000000000..d55c43cb741 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/databricks.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: deploy-postgres-replace-branch-$UNIQUE_NAME + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Replace existing branch test" + pg_version: 16 + history_retention_duration: "604800s" + + # Take over the implicitly-created production branch with replace_existing + # and apply a non-default spec (no_expiry: true). The field is input-only + # and not surfaced in get-branch, but it is visible in the recorded + # request body so the diff confirms it was sent. + postgres_branches: + production: + parent: ${resources.postgres_projects.my_project.id} + branch_id: production + replace_existing: true + no_expiry: true diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.destroy.direct.txt b/acceptance/bundle/resources/postgres_branches/replace_existing/out.destroy.direct.txt new file mode 100644 index 00000000000..03109f9364d --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.destroy.direct.txt @@ -0,0 +1,16 @@ +The following resources will be deleted: + delete resources.postgres_branches.production + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.production + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-replace-branch-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.destroy.terraform.txt b/acceptance/bundle/resources/postgres_branches/replace_existing/out.destroy.terraform.txt new file mode 100644 index 00000000000..03109f9364d --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.destroy.terraform.txt @@ -0,0 +1,16 @@ +The following resources will be deleted: + delete resources.postgres_branches.production + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.production + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-replace-branch-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.deploy.direct.json b/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.deploy.direct.json new file mode 100644 index 00000000000..781311e084c --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.deploy.direct.json @@ -0,0 +1,31 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Replace existing branch test", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches", + "q": { + "branch_id": "production", + "replace_existing": "true" + }, + "body": { + "spec": { + "no_expiry": true + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production" +} diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.deploy.terraform.json b/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.deploy.terraform.json new file mode 100644 index 00000000000..f70714acb2b --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.deploy.terraform.json @@ -0,0 +1,32 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Replace existing branch test", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches", + "q": { + "branch_id": "production", + "replace_existing": "true" + }, + "body": { + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "spec": { + "no_expiry": true + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production" +} diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.destroy.direct.json b/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.destroy.direct.json new file mode 100644 index 00000000000..1ddf631fc65 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.destroy.direct.json @@ -0,0 +1,47 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Replace existing branch test", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches", + "q": { + "branch_id": "production", + "replace_existing": "true" + }, + "body": { + "spec": { + "no_expiry": true + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.destroy.terraform.json b/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.destroy.terraform.json new file mode 100644 index 00000000000..ad6e8580927 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.requests.destroy.terraform.json @@ -0,0 +1,48 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "Replace existing branch test", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches", + "q": { + "branch_id": "production", + "replace_existing": "true" + }, + "body": { + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "spec": { + "no_expiry": true + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/production" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/output.txt b/acceptance/bundle/resources/postgres_branches/replace_existing/output.txt new file mode 100644 index 00000000000..3f1092385aa --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/output.txt @@ -0,0 +1,34 @@ + +>>> [CLI] bundle validate +Name: deploy-postgres-replace-branch-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-replace-branch-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-replace-branch-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] postgres get-branch projects/test-pg-proj-[UNIQUE_NAME]/branches/production +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/production", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "status": { + "branch_id": "production", + "current_state": "READY", + "default": true, + "is_protected": false, + "state_change_time": "[TIMESTAMP]" + }, + "uid": "[BRANCH_UID]" +} + +>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +=== bundle destroy +>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/script b/acceptance/bundle/resources/postgres_branches/replace_existing/script new file mode 100644 index 00000000000..3c3cc7e9300 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + # Belt-and-braces in case bundle destroy was skipped or partially failed. + $CLI postgres delete-project "projects/test-pg-proj-${UNIQUE_NAME}" 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +rm -f out.requests.txt +trace $CLI bundle deploy + +# Confirm the implicit production branch is now under management. +trace $CLI postgres get-branch "projects/test-pg-proj-${UNIQUE_NAME}/branches/production" | jq 'del(.create_time, .update_time, .status.logical_size_bytes)' + +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json + +# bundle destroy: the backend rejects independent deletion of the root +# branch with a MANAGED_BY_PARENT marker; both engines disregard that +# error and let the project delete cascade-clean the branch. Per-engine +# output is captured so regressions in either engine's destroy flow show +# up in the diff. +title "bundle destroy" +$CLI bundle destroy --auto-approve > out.destroy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true + +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/test.toml b/acceptance/bundle/resources/postgres_branches/replace_existing/test.toml new file mode 100644 index 00000000000..f8b3bbe49dd --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/test.toml @@ -0,0 +1 @@ +# All configuration inherited from parent test.toml diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_endpoints/replace_existing/databricks.yml.tmpl new file mode 100644 index 00000000000..220d7b0077e --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/databricks.yml.tmpl @@ -0,0 +1,43 @@ +bundle: + name: deploy-postgres-replace-endpoint-$UNIQUE_NAME + +sync: + paths: [] + +resources: + # The project's default_endpoint_settings are inherited by every implicit + # primary endpoint. We set suspend_timeout_duration to 300s here so the + # primary endpoint of the "develop" branch below would otherwise come up + # with 300s. + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Replace existing endpoint test" + pg_version: 16 + history_retention_duration: "604800s" + default_endpoint_settings: + autoscaling_limit_min_cu: 0.5 + autoscaling_limit_max_cu: 4 + suspend_timeout_duration: "300s" + + # Non-default branch managed normally — the server auto-provisions its + # primary read-write endpoint with the project defaults above. + postgres_branches: + develop: + parent: ${resources.postgres_projects.my_project.id} + branch_id: develop + no_expiry: true + + # Take over the implicitly-created primary endpoint of the develop branch + # and override the inherited suspend_timeout_duration (300s) with 600s. A + # successful deploy with 600s reflected in get-endpoint proves + # replace_existing applied the spec. + postgres_endpoints: + primary: + parent: ${resources.postgres_branches.develop.id} + endpoint_id: primary + replace_existing: true + endpoint_type: ENDPOINT_TYPE_READ_WRITE + autoscaling_limit_min_cu: 0.5 + autoscaling_limit_max_cu: 4 + suspend_timeout_duration: "600s" diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.destroy.direct.txt b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.destroy.direct.txt new file mode 100644 index 00000000000..2ac6d613c05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.destroy.direct.txt @@ -0,0 +1,17 @@ +The following resources will be deleted: + delete resources.postgres_branches.develop + delete resources.postgres_endpoints.primary + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.develop + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-replace-endpoint-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.destroy.terraform.txt b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.destroy.terraform.txt new file mode 100644 index 00000000000..2ac6d613c05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.destroy.terraform.txt @@ -0,0 +1,17 @@ +The following resources will be deleted: + delete resources.postgres_branches.develop + delete resources.postgres_endpoints.primary + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +This action will result in the deletion of the following Lakebase branches. +All data stored in them will be permanently lost: + delete resources.postgres_branches.develop + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-replace-endpoint-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.deploy.direct.json b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.deploy.direct.json new file mode 100644 index 00000000000..28e0d2433a9 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.deploy.direct.json @@ -0,0 +1,55 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Replace existing endpoint test", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches", + "q": { + "branch_id": "develop" + }, + "body": { + "spec": { + "no_expiry": true + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints", + "q": { + "endpoint_id": "primary", + "replace_existing": "true" + }, + "body": { + "spec": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_WRITE", + "suspend_timeout_duration": "600s" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.deploy.terraform.json b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.deploy.terraform.json new file mode 100644 index 00000000000..6ab68fc5dd9 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.deploy.terraform.json @@ -0,0 +1,53 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Replace existing endpoint test", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches", + "q": { + "branch_id": "develop" + }, + "body": { + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "spec": { + "no_expiry": true + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints", + "q": { + "endpoint_id": "primary", + "replace_existing": "true" + }, + "body": { + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/develop", + "spec": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_WRITE", + "suspend_timeout_duration": "600s" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.destroy.direct.json b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.destroy.direct.json new file mode 100644 index 00000000000..51e48eb6b96 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.destroy.direct.json @@ -0,0 +1,79 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Replace existing endpoint test", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches", + "q": { + "branch_id": "develop" + }, + "body": { + "spec": { + "no_expiry": true + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints", + "q": { + "endpoint_id": "primary", + "replace_existing": "true" + }, + "body": { + "spec": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_WRITE", + "suspend_timeout_duration": "600s" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.destroy.terraform.json b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.destroy.terraform.json new file mode 100644 index 00000000000..1bebfa32251 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.requests.destroy.terraform.json @@ -0,0 +1,77 @@ +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "default_endpoint_settings": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "suspend_timeout_duration": "300s" + }, + "display_name": "Replace existing endpoint test", + "history_retention_duration": "604800s", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches", + "q": { + "branch_id": "develop" + }, + "body": { + "parent": "projects/test-pg-proj-[UNIQUE_NAME]", + "spec": { + "no_expiry": true + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints", + "q": { + "endpoint_id": "primary", + "replace_existing": "true" + }, + "body": { + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/develop", + "spec": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "endpoint_type": "ENDPOINT_TYPE_READ_WRITE", + "suspend_timeout_duration": "600s" + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/develop" +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/output.txt b/acceptance/bundle/resources/postgres_endpoints/replace_existing/output.txt new file mode 100644 index 00000000000..c1823ef73c0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/output.txt @@ -0,0 +1,45 @@ + +>>> [CLI] bundle validate +Name: deploy-postgres-replace-endpoint-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-replace-endpoint-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-replace-endpoint-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] postgres get-endpoint projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary +{ + "name": "projects/test-pg-proj-[UNIQUE_NAME]/branches/develop/endpoints/primary", + "parent": "projects/test-pg-proj-[UNIQUE_NAME]/branches/develop", + "status": { + "autoscaling_limit_max_cu": 4, + "autoscaling_limit_min_cu": 0.5, + "current_state": "ACTIVE", + "disabled": false, + "endpoint_id": "primary", + "endpoint_type": "ENDPOINT_TYPE_READ_WRITE", + "group": { + "enable_readable_secondaries": false, + "max": 1, + "min": 1 + }, + "hosts": { + "host": "[ENDPOINT_UID].database.us-east-1.cloud.databricks.com" + }, + "settings": {}, + "suspend_timeout_duration": "600s" + }, + "uid": "[ENDPOINT_UID]" +} + +>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +=== bundle destroy +>>> print_requests.py --keep --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/script b/acceptance/bundle/resources/postgres_endpoints/replace_existing/script new file mode 100644 index 00000000000..b88120d5f31 --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/script @@ -0,0 +1,30 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + # Belt-and-braces in case bundle destroy was skipped or partially failed. + $CLI postgres delete-project "projects/test-pg-proj-${UNIQUE_NAME}" 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +rm -f out.requests.txt +trace $CLI bundle deploy + +# The implicit primary endpoint inherits suspend_timeout_duration from the +# project default (300s). After deploy with replace_existing=true, the +# bundle's spec must be applied: 600s. +trace $CLI postgres get-endpoint "projects/test-pg-proj-${UNIQUE_NAME}/branches/develop/endpoints/primary" | jq 'del(.create_time, .update_time)' + +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json + +# bundle destroy: the backend rejects independent deletion of the primary +# read-write endpoint with a MANAGED_BY_PARENT marker; both engines +# disregard that error and let the parent branch delete cascade-clean the +# endpoint. Per-engine output is captured so regressions in either +# engine's destroy flow show up in the diff. +title "bundle destroy" +$CLI bundle destroy --auto-approve > out.destroy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 || true + +trace print_requests.py --keep --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/test.toml b/acceptance/bundle/resources/postgres_endpoints/replace_existing/test.toml new file mode 100644 index 00000000000..f8b3bbe49dd --- /dev/null +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/test.toml @@ -0,0 +1 @@ +# All configuration inherited from parent test.toml diff --git a/bundle/config/resources/postgres_branch.go b/bundle/config/resources/postgres_branch.go index dd6e9daacdf..a5d093ca194 100644 --- a/bundle/config/resources/postgres_branch.go +++ b/bundle/config/resources/postgres_branch.go @@ -19,6 +19,11 @@ type PostgresBranchConfig struct { // Parent is the project containing this branch. Format: "projects/{project_id}" Parent string `json:"parent"` + + // ReplaceExisting, when true, takes over an existing branch with the same ID + // instead of returning ALREADY_EXISTS. Used to manage the implicitly-created + // production branch of a new project. Input-only: not returned by the GET API. + ReplaceExisting bool `json:"replace_existing,omitempty"` } func (c *PostgresBranchConfig) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/resources/postgres_endpoint.go b/bundle/config/resources/postgres_endpoint.go index 6dfe4fe7f33..1a9ce34665c 100644 --- a/bundle/config/resources/postgres_endpoint.go +++ b/bundle/config/resources/postgres_endpoint.go @@ -19,6 +19,11 @@ type PostgresEndpointConfig struct { // Parent is the branch containing this endpoint. Format: "projects/{project_id}/branches/{branch_id}" Parent string `json:"parent"` + + // ReplaceExisting, when true, takes over an existing endpoint with the same ID + // instead of returning ALREADY_EXISTS. Used to manage the implicitly-created + // primary read-write endpoint of a new branch. Input-only: not returned by the GET API. + ReplaceExisting bool `json:"replace_existing,omitempty"` } func (c *PostgresEndpointConfig) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_branch.go b/bundle/deploy/terraform/tfdyn/convert_postgres_branch.go index c1d598a1e64..12914e93074 100644 --- a/bundle/deploy/terraform/tfdyn/convert_postgres_branch.go +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_branch.go @@ -15,7 +15,7 @@ func (c postgresBranchConverter) Convert(ctx context.Context, key string, vin dy // The bundle config has flattened BranchSpec fields at the top level. // Terraform expects them nested in a "spec" block. specFields := specFieldNames(schema.ResourcePostgresBranchSpec{}) - topLevelFields := []string{"branch_id", "parent"} + topLevelFields := []string{"branch_id", "parent", "replace_existing"} // Build the spec block from the flattened fields specMap := make(map[string]dyn.Value) diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_endpoint.go b/bundle/deploy/terraform/tfdyn/convert_postgres_endpoint.go index f97582a2a33..b603abfcb75 100644 --- a/bundle/deploy/terraform/tfdyn/convert_postgres_endpoint.go +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_endpoint.go @@ -15,7 +15,7 @@ func (c postgresEndpointConverter) Convert(ctx context.Context, key string, vin // The bundle config has flattened EndpointSpec fields at the top level. // Terraform expects them nested in a "spec" block. specFields := specFieldNames(schema.ResourcePostgresEndpointSpec{}) - topLevelFields := []string{"endpoint_id", "parent"} + topLevelFields := []string{"endpoint_id", "parent", "replace_existing"} // Build the spec block from the flattened fields specMap := make(map[string]dyn.Value) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index ba0f54bfc7b..e327cb8563e 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -80,9 +80,12 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, } func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentState, oldID string, newState any) error { - // Note, unlike Delete(), we hard error on 403 here intentionally + // Note, unlike Delete(), we hard error on 403 here intentionally. + // MANAGED_BY_PARENT is still disregarded — the subsequent Create with + // replace_existing=true will reconfigure the parent-managed resource in + // place, matching the Terraform provider's recreate behaviour. err := d.Adapter.DoDelete(ctx, oldID) - if err != nil && !isResourceGone(err) { + if err != nil && !isResourceGone(err) && !isManagedByParent(err) { return fmt.Errorf("deleting old id=%s: %w", oldID, err) } @@ -168,7 +171,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, oldID string) error { err := d.Adapter.DoDelete(ctx, oldID) - if err != nil && !isResourceGone(err) { + if err != nil && !isResourceGone(err) && !isManagedByParent(err) { // Rather than failing delete and requiring user to unbind, we perform unbind automatically there. // Some services, e.g. jobs, return 403 for missing resources if caller did not have permissions to it when job existed. // In those cases 403 hides 404. In other cases, user not having permissions to resource but having in the bundle might diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index 19aa77ce7f1..be8833acc95 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -21,9 +21,10 @@ func (*ResourcePostgresBranch) New(client *databricks.WorkspaceClient) *Resource func (*ResourcePostgresBranch) PrepareState(input *resources.PostgresBranch) *PostgresBranchState { return &PostgresBranchState{ - BranchId: input.BranchId, - Parent: input.Parent, - BranchSpec: input.BranchSpec, + BranchId: input.BranchId, + Parent: input.Parent, + ReplaceExisting: input.ReplaceExisting, + BranchSpec: input.BranchSpec, } } @@ -36,6 +37,10 @@ func (*ResourcePostgresBranch) RemapState(remote *postgres.Branch) *PostgresBran BranchId: components.BranchID, Parent: remote.Parent, + // replace_existing is a create-time-only flag; the GET API never returns + // it, so RemapState leaves it false. + ReplaceExisting: false, + // The read API does not return the spec, only the status. // This means we cannot detect remote drift for spec fields. // Use an empty struct (not nil) so field-level diffing works correctly. @@ -72,7 +77,7 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB UpdateTime: nil, ForceSendFields: nil, }, - ReplaceExisting: false, + ReplaceExisting: config.ReplaceExisting, ForceSendFields: nil, }) if err != nil { diff --git a/bundle/direct/dresources/postgres_endpoint.go b/bundle/direct/dresources/postgres_endpoint.go index 9ccec27f327..14698d7de2b 100644 --- a/bundle/direct/dresources/postgres_endpoint.go +++ b/bundle/direct/dresources/postgres_endpoint.go @@ -30,9 +30,10 @@ func (*ResourcePostgresEndpoint) New(client *databricks.WorkspaceClient) *Resour func (*ResourcePostgresEndpoint) PrepareState(input *resources.PostgresEndpoint) *PostgresEndpointState { return &PostgresEndpointState{ - EndpointId: input.EndpointId, - Parent: input.Parent, - EndpointSpec: input.EndpointSpec, + EndpointId: input.EndpointId, + Parent: input.Parent, + ReplaceExisting: input.ReplaceExisting, + EndpointSpec: input.EndpointSpec, } } @@ -45,6 +46,10 @@ func (*ResourcePostgresEndpoint) RemapState(remote *postgres.Endpoint) *Postgres EndpointId: components.EndpointID, Parent: remote.Parent, + // replace_existing is a create-time-only flag; the GET API never returns + // it, so RemapState leaves it false. + ReplaceExisting: false, + // The read API does not return the spec, only the status. // This means we cannot detect remote drift for spec fields. // Use an empty struct (not nil) so field-level diffing works correctly. @@ -112,7 +117,7 @@ func (r *ResourcePostgresEndpoint) DoCreate(ctx context.Context, config *Postgre UpdateTime: nil, ForceSendFields: nil, }, - ReplaceExisting: false, + ReplaceExisting: config.ReplaceExisting, ForceSendFields: nil, }) if err != nil { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 569fca9ee82..4eb5933ad25 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -499,6 +499,14 @@ resources: reason: immutable - field: branch_id reason: immutable + ignore_remote_changes: + # replace_existing is a create-time query parameter; not returned by GET. + - field: replace_existing + reason: input_only + ignore_local_changes: + # replace_existing only takes effect on create; toggling it later is a no-op. + - field: replace_existing + reason: "input_only; cannot be updated after create" postgres_endpoints: recreate_on_changes: @@ -507,6 +515,14 @@ resources: reason: immutable - field: endpoint_id reason: immutable + ignore_remote_changes: + # replace_existing is a create-time query parameter; not returned by GET. + - field: replace_existing + reason: input_only + ignore_local_changes: + # replace_existing only takes effect on create; toggling it later is a no-op. + - field: replace_existing + reason: "input_only; cannot be updated after create" vector_search_endpoints: recreate_on_changes: diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 88f246723bf..d666b966f04 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -48,6 +48,7 @@ var knownMissingInRemoteType = map[string][]string{ "expire_time", "is_protected", "no_expiry", + "replace_existing", "source_branch", "source_branch_lsn", "source_branch_time", @@ -61,6 +62,7 @@ var knownMissingInRemoteType = map[string][]string{ "endpoint_type", "group", "no_suspension", + "replace_existing", "settings", "suspend_timeout_duration", }, diff --git a/bundle/direct/util.go b/bundle/direct/util.go index 43c39c9a67f..3b4c9abd651 100644 --- a/bundle/direct/util.go +++ b/bundle/direct/util.go @@ -9,3 +9,19 @@ import ( func isResourceGone(err error) bool { return errors.Is(err, apierr.ErrResourceDoesNotExist) || errors.Is(err, apierr.ErrNotFound) } + +// isManagedByParent reports whether err is an API error carrying the +// declarative_context=MANAGED_BY_PARENT marker in ErrorInfo.metadata. The +// server uses this to signal that a resource's lifecycle is owned by a +// parent (e.g. a Lakebase RW endpoint inside a branch, or a root branch +// inside a project) and the standalone Delete can be safely disregarded — +// the parent's Delete will cascade-clean. Mirrors the TF provider's +// declarative.IsDeleteError suppression. +func isManagedByParent(err error) bool { + var apiErr *apierr.APIError + if !errors.As(err, &apiErr) || apiErr == nil { + return false + } + info := apiErr.ErrorDetails().ErrorInfo + return info != nil && info.Metadata["declarative_context"] == "MANAGED_BY_PARENT" +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f6ac5c45d4d..f15766a0d3b 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -769,6 +769,9 @@ github.com/databricks/cli/bundle/config/resources.PostgresBranch: "parent": "description": |- PLACEHOLDER + "replace_existing": + "description": |- + PLACEHOLDER "source_branch": "description": |- PLACEHOLDER @@ -827,6 +830,9 @@ github.com/databricks/cli/bundle/config/resources.PostgresEndpoint: "parent": "description": |- PLACEHOLDER + "replace_existing": + "description": |- + PLACEHOLDER "settings": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 26e773f64a4..ac57cedbe53 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1384,6 +1384,9 @@ "parent": { "$ref": "#/$defs/string" }, + "replace_existing": { + "$ref": "#/$defs/bool" + }, "source_branch": { "$ref": "#/$defs/string" }, @@ -1441,6 +1444,9 @@ "parent": { "$ref": "#/$defs/string" }, + "replace_existing": { + "$ref": "#/$defs/bool" + }, "settings": { "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/postgres.EndpointSettings" }, diff --git a/libs/testproxy/server.go b/libs/testproxy/server.go index f3510d7adb0..fd6038c8ed5 100644 --- a/libs/testproxy/server.go +++ b/libs/testproxy/server.go @@ -2,7 +2,6 @@ package testproxy import ( "bytes" - "encoding/json" "errors" "net/http" "net/http/httptest" @@ -129,22 +128,31 @@ func (s *ProxyServer) proxyToCloud(w http.ResponseWriter, r *http.Request) { var encodedResponse *testserver.EncodedResponse - // API errors from the SDK are expected to be of the type [apierr.APIError]. If we - // get an API error then parse the error and forward it back to the client - // in an appropriate format. + // API errors from the SDK are of type [apierr.APIError]. Forward the raw + // response bytes verbatim — including any error details — so callers see + // exactly what the workspace returned. Re-marshalling from the parsed + // APIError would drop fields the SDK doesn't surface (e.g. metadata in + // details[]) and silently break callers that inspect them. apiErr := &apierr.APIError{} if errors.As(err, &apiErr) { - body := map[string]string{ - "error_code": apiErr.ErrorCode, - "message": apiErr.Message, + rw := apiErr.ResponseWrapper + if rw == nil { + // The SDK populates ResponseWrapper for every APIError produced + // from a real HTTP response. If this ever fires the SDK changed + // shape and we need to revisit how we forward error bodies. + panic("apierr.APIError has no ResponseWrapper") } - - b, err := json.Marshal(body) - assert.NoError(s.t, err) - encodedResponse = &testserver.EncodedResponse{ - StatusCode: apiErr.StatusCode, - Body: b, + StatusCode: rw.Response.StatusCode, + Body: rw.DebugBytes, + } + // Visitors registered via WithResponseHeader are not invoked when + // the SDK returns an error, so populate the include list directly + // from the original response headers. + for _, header := range includeResponseHeaders { + if v := rw.Response.Header.Get(header); v != "" { + *responseHeaders[header] = v + } } } diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 5430c68cbcc..6fc686c4fd3 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -173,6 +173,13 @@ type FakeWorkspace struct { PostgresEndpoints map[string]postgres.Endpoint PostgresOperations map[string]postgres.Operation + // Branches and endpoints that the server provisioned implicitly together + // with their parent (e.g. the production branch on a new project, or the + // primary endpoint on a new branch). The real backend rejects independent + // deletion of these — they go away only when the parent is deleted. + postgresImplicitBranches map[string]bool + postgresImplicitEndpoints map[string]bool + // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. clusterVenvs map[string]*clusterEnv @@ -285,26 +292,28 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { State: sql.StateRunning, }, }, - ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, - VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, - Repos: map[string]workspace.RepoInfo{}, - SecretScopes: map[string]workspace.SecretScope{}, - Secrets: map[string]map[string]string{}, - Acls: map[string][]workspace.AclItem{}, - Permissions: map[string]iam.ObjectPermissions{}, - Groups: map[string]iam.Group{}, - DatabaseInstances: map[string]database.DatabaseInstance{}, - DatabaseCatalogs: map[string]database.DatabaseCatalog{}, - SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, - PostgresProjects: map[string]postgres.Project{}, - PostgresBranches: map[string]postgres.Branch{}, - PostgresEndpoints: map[string]postgres.Endpoint{}, - PostgresOperations: map[string]postgres.Operation{}, - clusterVenvs: map[string]*clusterEnv{}, - Alerts: map[string]sql.AlertV2{}, - Experiments: map[string]ml.GetExperimentResponse{}, - ModelRegistryModels: map[string]ml.Model{}, - ModelRegistryModelIDs: map[string]string{}, + ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, + VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, + Repos: map[string]workspace.RepoInfo{}, + SecretScopes: map[string]workspace.SecretScope{}, + Secrets: map[string]map[string]string{}, + Acls: map[string][]workspace.AclItem{}, + Permissions: map[string]iam.ObjectPermissions{}, + Groups: map[string]iam.Group{}, + DatabaseInstances: map[string]database.DatabaseInstance{}, + DatabaseCatalogs: map[string]database.DatabaseCatalog{}, + SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, + PostgresProjects: map[string]postgres.Project{}, + PostgresBranches: map[string]postgres.Branch{}, + PostgresEndpoints: map[string]postgres.Endpoint{}, + PostgresOperations: map[string]postgres.Operation{}, + postgresImplicitBranches: map[string]bool{}, + postgresImplicitEndpoints: map[string]bool{}, + clusterVenvs: map[string]*clusterEnv{}, + Alerts: map[string]sql.AlertV2{}, + Experiments: map[string]ml.GetExperimentResponse{}, + ModelRegistryModels: map[string]ml.Model{}, + ModelRegistryModelIDs: map[string]string{}, Clusters: map[string]compute.ClusterDetails{ TestDefaultClusterId: { ClusterId: TestDefaultClusterId, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index d98011fc7ba..8a1188a79ec 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -886,7 +886,8 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches", func(req Request) any { parent := "projects/" + req.Vars["project_id"] branchID := req.URL.Query().Get("branch_id") - return req.Workspace.PostgresBranchCreate(req, parent, branchID) + replaceExisting := req.URL.Query().Get("replace_existing") == "true" + return req.Workspace.PostgresBranchCreate(req, parent, branchID, replaceExisting) }) server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches", func(req Request) any { @@ -913,7 +914,8 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/endpoints", func(req Request) any { parent := "projects/" + req.Vars["project_id"] + "/branches/" + req.Vars["branch_id"] endpointID := req.URL.Query().Get("endpoint_id") - return req.Workspace.PostgresEndpointCreate(req, parent, endpointID) + replaceExisting := req.URL.Query().Get("replace_existing") == "true" + return req.Workspace.PostgresEndpointCreate(req, parent, endpointID, replaceExisting) }) server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/endpoints", func(req Request) any { diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index f3a488b5704..8a7c22b4ecf 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -26,6 +26,36 @@ func postgresErrorResponse(statusCode int, errorCode, message string) Response { } } +// postgresManagedByParentErrorResponse creates a 400 BAD_REQUEST response that +// mirrors the real backend's "managed by parent" payload — including the +// details[].ErrorInfo.metadata.declarative_context=MANAGED_BY_PARENT marker +// that declarative clients (TF provider, direct engine) use to disregard the +// delete and rely on the parent to cascade. +func postgresManagedByParentErrorResponse(message string) Response { + return Response{ + StatusCode: 400, + Body: map[string]any{ + "error_code": "BAD_REQUEST", + "message": message, + "details": []any{ + map[string]any{ + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "domain": "databricks.com", + "reason": "RESOURCE_CONFLICT", + "metadata": map[string]string{ + "declarative_context": "MANAGED_BY_PARENT", + }, + }, + map[string]any{ + "@type": "type.googleapis.com/google.rpc.RequestInfo", + "request_id": nextUUID(), + "serving_data": "", + }, + }, + }, + } +} + // postgresNotFoundResponse creates a NOT_FOUND error response for a resource type. func postgresNotFoundResponse(resourceType string) Response { // Include trace ID to match real API behavior for not found errors @@ -192,6 +222,22 @@ func (s *FakeWorkspace) PostgresProjectDelete(name string) Response { return postgresNotFoundResponse("project") } + // Deleting a project cascade-deletes all its branches and endpoints. + branchPrefix := name + "/branches/" + for brName := range s.PostgresBranches { + if strings.HasPrefix(brName, branchPrefix) { + delete(s.PostgresBranches, brName) + delete(s.postgresImplicitBranches, brName) + endpointPrefix := brName + "/endpoints/" + for epName := range s.PostgresEndpoints { + if strings.HasPrefix(epName, endpointPrefix) { + delete(s.PostgresEndpoints, epName) + delete(s.postgresImplicitEndpoints, epName) + } + } + } + } + delete(s.PostgresProjects, name) return Response{ @@ -200,7 +246,11 @@ func (s *FakeWorkspace) PostgresProjectDelete(name string) Response { } // PostgresBranchCreate creates a new postgres branch. -func (s *FakeWorkspace) PostgresBranchCreate(req Request, parent, branchID string) Response { +// +// When replaceExisting is true, an existing branch with the same ID is updated +// in place instead of returning ALREADY_EXISTS. This mirrors the backend behavior +// that lets users bring the implicitly-created production branch under management. +func (s *FakeWorkspace) PostgresBranchCreate(req Request, parent, branchID string, replaceExisting bool) Response { defer s.LockUnlock()() // Validate that branch_id is provided @@ -225,44 +275,51 @@ func (s *FakeWorkspace) PostgresBranchCreate(req Request, parent, branchID strin name := fmt.Sprintf("%s/branches/%s", parent, branchID) - // Check for duplicate - if _, exists := s.PostgresBranches[name]; exists { + existing, exists := s.PostgresBranches[name] + if exists && !replaceExisting { return postgresErrorResponse(409, "ALREADY_EXISTS", "branch with such id already exists") } now := nowTime() - branch.Name = name - branch.Parent = parent - branch.Uid = "br-" + nextUUID()[:20] - branch.CreateTime = now - branch.UpdateTime = now - - // Find the default branch to use as source - var defaultBranch *postgres.Branch - prefix := parent + "/branches/" - for brName, br := range s.PostgresBranches { - if strings.HasPrefix(brName, prefix) && br.Status != nil && br.Status.Default { - defaultBranch = &br - break + if exists { + // Preserve identifying / output-only fields; apply incoming spec to status. + branch.Name = existing.Name + branch.Parent = existing.Parent + branch.Uid = existing.Uid + branch.CreateTime = existing.CreateTime + branch.UpdateTime = now + branch.Status = existing.Status + } else { + branch.Name = name + branch.Parent = parent + branch.Uid = "br-" + nextUUID()[:20] + branch.CreateTime = now + branch.UpdateTime = now + branch.Status = &postgres.BranchStatus{ + BranchId: branchID, + CurrentState: postgres.BranchStatusStateReady, + StateChangeTime: now, + Default: false, + IsProtected: false, + LogicalSizeBytes: 0, + ForceSendFields: []string{"Default", "IsProtected", "LogicalSizeBytes"}, } - } - // Initialize status with all required fields - branch.Status = &postgres.BranchStatus{ - BranchId: branchID, - CurrentState: postgres.BranchStatusStateReady, - StateChangeTime: now, - Default: false, - IsProtected: false, - LogicalSizeBytes: 0, - ForceSendFields: []string{"Default", "IsProtected", "LogicalSizeBytes"}, + // Set source branch info if a default branch exists. + prefix := parent + "/branches/" + for brName, br := range s.PostgresBranches { + if strings.HasPrefix(brName, prefix) && br.Status != nil && br.Status.Default { + branch.Status.SourceBranch = br.Name + branch.Status.SourceBranchLsn = "0/0" + branch.Status.SourceBranchTime = nowTime() + break + } + } } - // Set source branch info if a default branch exists - if defaultBranch != nil { - branch.Status.SourceBranch = defaultBranch.Name - branch.Status.SourceBranchLsn = "0/0" - branch.Status.SourceBranchTime = nowTime() + // Apply user-provided spec fields to status (where input fields are surfaced). + if branch.Spec != nil { + branch.Status.IsProtected = branch.Spec.IsProtected } // Clear spec - API only returns status @@ -270,6 +327,12 @@ func (s *FakeWorkspace) PostgresBranchCreate(req Request, parent, branchID strin s.PostgresBranches[name] = branch + // Each branch implicitly provisions a primary read-write endpoint. Only create + // it on first branch creation; on replace_existing the primary already exists. + if !exists { + s.createDefaultEndpointLocked(name) + } + return Response{ Body: s.createOperationLocked(branch.Name, branch), } @@ -366,12 +429,30 @@ func (s *FakeWorkspace) PostgresBranchDelete(name string) Response { return postgresNotFoundResponse("branch") } - // Check if branch is protected if branch.Status != nil && branch.Status.IsProtected { return postgresErrorResponse(400, "BAD_REQUEST", "cannot delete protected branch") } + // Branches the server provisioned implicitly (the project's root branch) + // cannot be deleted independently; they are removed when the project is + // deleted. Matches the real backend's error payload including the + // declarative_context=MANAGED_BY_PARENT marker. + if s.postgresImplicitBranches[name] { + return postgresManagedByParentErrorResponse(fmt.Sprintf("Cannot delete root branch '%s'. Root branches cannot be deleted independently.", name)) + } + + // Deleting a branch cascade-deletes its endpoints (including the implicit + // primary). + endpointPrefix := name + "/endpoints/" + for epName := range s.PostgresEndpoints { + if strings.HasPrefix(epName, endpointPrefix) { + delete(s.PostgresEndpoints, epName) + delete(s.postgresImplicitEndpoints, epName) + } + } + delete(s.PostgresBranches, name) + delete(s.postgresImplicitBranches, name) return Response{ Body: s.createOperationLocked(name, nil), @@ -379,7 +460,12 @@ func (s *FakeWorkspace) PostgresBranchDelete(name string) Response { } // PostgresEndpointCreate creates a new postgres endpoint. -func (s *FakeWorkspace) PostgresEndpointCreate(req Request, parent, endpointID string) Response { +// +// When replaceExisting is true, an existing endpoint with the same ID is updated +// in place instead of returning ALREADY_EXISTS. This mirrors the backend behavior +// that lets users bring the implicitly-created primary read-write endpoint under +// management. +func (s *FakeWorkspace) PostgresEndpointCreate(req Request, parent, endpointID string, replaceExisting bool) Response { defer s.LockUnlock()() // Validate that endpoint_id is provided @@ -405,37 +491,56 @@ func (s *FakeWorkspace) PostgresEndpointCreate(req Request, parent, endpointID s name := fmt.Sprintf("%s/endpoints/%s", parent, endpointID) - // Check for duplicate - if _, exists := s.PostgresEndpoints[name]; exists { + existing, exists := s.PostgresEndpoints[name] + if exists && !replaceExisting { return postgresErrorResponse(409, "ALREADY_EXISTS", "endpoint with such id already exists") } now := nowTime() - endpoint.Name = name - endpoint.Parent = parent - endpoint.Uid = "ep-" + nextUUID()[:20] - endpoint.CreateTime = now - endpoint.UpdateTime = now - - // Get default suspend timeout from project - var defaultSuspendTimeout *duration.Duration - if project, ok := s.PostgresProjects[branch.Parent]; ok { - if project.Status != nil && project.Status.DefaultEndpointSettings != nil { - defaultSuspendTimeout = project.Status.DefaultEndpointSettings.SuspendTimeoutDuration + if exists { + endpoint.Name = existing.Name + endpoint.Parent = existing.Parent + endpoint.Uid = existing.Uid + endpoint.CreateTime = existing.CreateTime + endpoint.UpdateTime = now + endpoint.Status = existing.Status + } else { + endpoint.Name = name + endpoint.Parent = parent + endpoint.Uid = "ep-" + nextUUID()[:20] + endpoint.CreateTime = now + endpoint.UpdateTime = now + + // Inherit suspend timeout default from the project. + var defaultSuspendTimeout *duration.Duration + if project, ok := s.PostgresProjects[branch.Parent]; ok { + if project.Status != nil && project.Status.DefaultEndpointSettings != nil { + defaultSuspendTimeout = project.Status.DefaultEndpointSettings.SuspendTimeoutDuration + } + } + + endpoint.Status = &postgres.EndpointStatus{ + EndpointId: endpointID, + CurrentState: postgres.EndpointStatusStateActive, + Disabled: false, + Settings: &postgres.EndpointSettings{}, + SuspendTimeoutDuration: defaultSuspendTimeout, + ForceSendFields: []string{"Disabled"}, } - } - // Initialize status with all required fields - endpoint.Status = &postgres.EndpointStatus{ - EndpointId: endpointID, - CurrentState: postgres.EndpointStatusStateActive, - Disabled: false, - Settings: &postgres.EndpointSettings{}, - SuspendTimeoutDuration: defaultSuspendTimeout, - ForceSendFields: []string{"Disabled"}, + host := endpoint.Uid + ".database.us-east-1.cloud.databricks.com" + endpoint.Status.Hosts = &postgres.EndpointHosts{ + Host: host, + ReadOnlyHost: host, + } + endpoint.Status.Group = &postgres.EndpointGroupStatus{ + EnableReadableSecondaries: true, + Max: 1, + Min: 1, + } } - // Copy spec values to status + // Apply user-provided spec to status (where input fields are surfaced). if endpoint.Spec != nil { endpoint.Status.EndpointType = endpoint.Spec.EndpointType endpoint.Status.AutoscalingLimitMinCu = endpoint.Spec.AutoscalingLimitMinCu @@ -443,23 +548,7 @@ func (s *FakeWorkspace) PostgresEndpointCreate(req Request, parent, endpointID s if endpoint.Spec.SuspendTimeoutDuration != nil { endpoint.Status.SuspendTimeoutDuration = endpoint.Spec.SuspendTimeoutDuration } - if endpoint.Spec.Disabled { - endpoint.Status.Disabled = true - } - } - - // Generate fake hosts - host := endpoint.Uid + ".database.us-east-1.cloud.databricks.com" - endpoint.Status.Hosts = &postgres.EndpointHosts{ - Host: host, - ReadOnlyHost: host, - } - - // Set default group status - endpoint.Status.Group = &postgres.EndpointGroupStatus{ - EnableReadableSecondaries: true, - Max: 1, - Min: 1, + endpoint.Status.Disabled = endpoint.Spec.Disabled } // Clear spec - API only returns status @@ -580,7 +669,16 @@ func (s *FakeWorkspace) PostgresEndpointDelete(name string) Response { return postgresNotFoundResponse("endpoint") } + // Endpoints the server provisioned implicitly (the primary read-write + // endpoint on every branch) cannot be deleted independently; they are + // removed when the parent branch is deleted. Matches the real backend's + // error payload including the declarative_context=MANAGED_BY_PARENT marker. + if s.postgresImplicitEndpoints[name] { + return postgresManagedByParentErrorResponse(fmt.Sprintf("Cannot delete read-write endpoint '%s'. Read-write endpoints cannot be deleted.", name)) + } + delete(s.PostgresEndpoints, name) + delete(s.postgresImplicitEndpoints, name) return Response{ Body: s.createOperationLocked(name, nil), @@ -656,4 +754,61 @@ func (s *FakeWorkspace) createDefaultBranchLocked(projectName string) { } s.PostgresBranches[branchName] = defaultBranch + s.postgresImplicitBranches[branchName] = true + + // Each branch implicitly provisions a primary read-write endpoint. + s.createDefaultEndpointLocked(branchName) +} + +// createDefaultEndpointLocked creates the implicit primary read-write endpoint for +// a branch (caller must hold lock). The endpoint is named "primary" to match cloud +// API behavior. +func (s *FakeWorkspace) createDefaultEndpointLocked(branchName string) { + now := nowTime() + endpointName := branchName + "/endpoints/primary" + + // Inherit default endpoint settings from the project where available. + var ( + minCu, maxCu float64 + suspendTimeout *duration.Duration + projectName = strings.Split(branchName, "/branches/")[0] + ) + if project, ok := s.PostgresProjects[projectName]; ok { + if project.Status != nil && project.Status.DefaultEndpointSettings != nil { + minCu = project.Status.DefaultEndpointSettings.AutoscalingLimitMinCu + maxCu = project.Status.DefaultEndpointSettings.AutoscalingLimitMaxCu + suspendTimeout = project.Status.DefaultEndpointSettings.SuspendTimeoutDuration + } + } + + endpointUID := "ep-" + nextUUID()[:20] + host := endpointUID + ".database.us-east-1.cloud.databricks.com" + + s.PostgresEndpoints[endpointName] = postgres.Endpoint{ + Name: endpointName, + Parent: branchName, + Uid: endpointUID, + CreateTime: now, + UpdateTime: now, + Status: &postgres.EndpointStatus{ + EndpointId: "primary", + EndpointType: postgres.EndpointTypeEndpointTypeReadWrite, + CurrentState: postgres.EndpointStatusStateActive, + Disabled: false, + Settings: &postgres.EndpointSettings{}, + AutoscalingLimitMinCu: minCu, + AutoscalingLimitMaxCu: maxCu, + SuspendTimeoutDuration: suspendTimeout, + // Primary read-write endpoint exposes only the read-write host; + // read_only_host is set on read-only endpoints created later. + Hosts: &postgres.EndpointHosts{Host: host}, + Group: &postgres.EndpointGroupStatus{ + Max: 1, + Min: 1, + ForceSendFields: []string{"EnableReadableSecondaries"}, + }, + ForceSendFields: []string{"Disabled"}, + }, + } + s.postgresImplicitEndpoints[endpointName] = true } diff --git a/libs/testserver/postgres_test.go b/libs/testserver/postgres_test.go index d421212ed9c..fa8a4eed35d 100644 --- a/libs/testserver/postgres_test.go +++ b/libs/testserver/postgres_test.go @@ -234,9 +234,11 @@ func TestPostgresEndpointCRUD(t *testing.T) { require.NoError(t, err) assert.Equal(t, 200, listEpResp.StatusCode) + // Each branch implicitly provisions a "primary" RW endpoint in addition to + // the explicitly-created "rw-endpoint". var listEndpoints postgres.ListEndpointsResponse require.NoError(t, json.NewDecoder(listEpResp.Body).Decode(&listEndpoints)) - assert.Len(t, listEndpoints.Endpoints, 1) + assert.Len(t, listEndpoints.Endpoints, 2) listEpResp.Body.Close() // Delete endpoint