[WIP] Investigate intermittent failures of jobs

acceptance/bin/fault.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env python3
 """Set up a fault rule on the testserver for the current test token.
 
-Usage: fault.py PATTERN STATUS_CODE OFFSET TIMES
+Usage: fault.py PATTERN STATUS_CODE OFFSET TIMES [DELAY_MS]
 
   PATTERN     HTTP method and path, supports trailing * wildcard,
               e.g. "PUT /api/2.0/permissions/pipelines/*"
-  STATUS_CODE HTTP status code to return, e.g. 504
+  STATUS_CODE HTTP status code to return, e.g. 504. Use 0 together with
+              DELAY_MS for a delay-only rule that sleeps and then lets the
+              real handler run (a slow-but-successful response).
   OFFSET      number of requests to let through before fault starts
-  TIMES       number of times to return the fault response
+  TIMES       number of times to apply the rule
+  DELAY_MS    optional milliseconds to sleep before responding (default 0)
 
 The rule is scoped to the current DATABRICKS_TOKEN so it only affects
 the test that registers it, even when tests share a server.
@@ -25,18 +28,22 @@
     print("DATABRICKS_HOST not set", file=sys.stderr)
     sys.exit(1)
 
-if len(sys.argv) != 5:
-    print(f"usage: {sys.argv[0]} PATTERN STATUS_CODE OFFSET TIMES", file=sys.stderr)
+if len(sys.argv) not in (5, 6):
+    print(f"usage: {sys.argv[0]} PATTERN STATUS_CODE OFFSET TIMES [DELAY_MS]", file=sys.stderr)
     sys.exit(1)
 
 pattern, status_code, offset, times = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])
-body = '{"error_code":"INJECTED","message":"Fault injected by test."}'
+delay_ms = int(sys.argv[5]) if len(sys.argv) == 6 else 0
+# A delay-only rule (status_code 0) has no error body; the testserver falls
+# through to the real handler after sleeping.
+body = "" if status_code == 0 else '{"error_code":"INJECTED","message":"Fault injected by test."}'
 
 data = json.dumps(
     {
         "pattern": pattern,
         "status_code": status_code,
         "body": body,
+        "delay_ms": delay_ms,
         "offset": offset,
         "times": times,
     }

acceptance/bundle/deploy/deploy-race-repro/gap/app/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: deploy-race-gap
+
+resources:
+  jobs:
+    etl:
+      name: etl
+      tasks:
+        - task_key: run_etl
+          spark_python_task:
+            python_file: ./src/etl.py
+          environment_key: default
+      environments:
+        - environment_key: default
+          spec:
+            client: "2"

acceptance/bundle/deploy/deploy-race-repro/gap/app/src/etl.py

@@ -0,0 +1 @@
+print("etl task ran")

acceptance/bundle/deploy/deploy-race-repro/gap/script

@@ -0,0 +1,46 @@
+# THROWAWAY investigation harness -- NOT a CI test (disabled via parent test.toml).
+#
+# Behaviour 1: delete-before-put gap raced by a live job.
+#
+# Renaming a task file makes applyDiff DELETE the old remote path and PUT the new
+# one (libs/sync/diff.go addFilesWithRemoteNameChanged). Because the job is only
+# repointed after files.Upload (bundle/phases/deploy.go), a job that runs during
+# the gap sees a transient "file not found". This proves it as a real wall-clock
+# race by delaying the PUT and hammering the job while the window is open.
+#
+# Run it by hand to observe the failure:
+#   go test ./acceptance -run TestAccept/bundle/deploy/deploy-race-repro/gap \
+#     -update -tail -v
+
+cd app
+
+unset MSYS_NO_PATHCONV
+
+trace $CLI bundle deploy
+trace $CLI bundle run etl
+
+FILES_PATH=$($CLI bundle summary --output json | jq -r '.workspace.file_path')
+
+# Rename the task file: applyDiff DELETEs .../src/etl.py, then PUTs .../src/etl_v2.py.
+# Delay that PUT by 3s so the delete-before-put window stays open long enough for a
+# concurrently running job to fall into it.
+mv src/etl.py src/etl_v2.py
+update_file.py databricks.yml "etl.py" "etl_v2.py"
+fault.py "POST /api/2.0/workspace-files/import-file${FILES_PATH}/src/etl_v2.py" 0 0 1 3000
+
+# Start the slow deploy in the background, then hammer the job with runs while the
+# window is open. At least one run should observe the transient "file not found".
+$CLI bundle deploy &> LOG.deploy &
+DEPLOY_PID=$!
+
+for i in $(seq 1 20); do
+    echo ">>> run attempt $i" >> LOG.runs
+    $CLI bundle run etl &>> LOG.runs || echo "RUN $i FAILED (raced the deploy gap)" >> LOG.runs
+done
+
+wait $DEPLOY_PID
+
+echo "--- runs that raced the deploy (transient not-found) ---" >> LOG.summary
+grep -c "FAILED (raced the deploy gap)" LOG.runs >> LOG.summary || true
+
+trace $CLI bundle destroy --auto-approve &>> LOG.destroy

acceptance/bundle/deploy/deploy-race-repro/permissions-reset/app/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: deploy-race-permissions
+
+resources:
+  jobs:
+    etl:
+      name: etl
+      tasks:
+        - task_key: run_etl
+          spark_python_task:
+            python_file: ./src/etl.py
+          environment_key: default
+      environments:
+        - environment_key: default
+          spec:
+            client: "2"

acceptance/bundle/deploy/deploy-race-repro/permissions-reset/app/src/etl.py

@@ -0,0 +1 @@
+print("etl task ran")

acceptance/bundle/deploy/deploy-race-repro/permissions-reset/script

@@ -0,0 +1,34 @@
+# THROWAWAY investigation harness -- NOT a CI test (disabled via parent test.toml).
+#
+# Behaviour 3: permission ACL reset on the deployed folder after upload.
+#
+# ApplyWorkspaceRootPermissions runs SetPermissions AFTER files.Upload
+# (bundle/phases/deploy.go), replacing the folder's direct ACL with exactly the
+# declared permissions. Recording the calls shows the run-as SP's access is
+# rewritten on every deploy, not merely added to -- so a deploy can transiently
+# narrow access rather than only widening it.
+#
+# Run it by hand to observe the recorded ACL writes:
+#   go test ./acceptance -run TestAccept/bundle/deploy/deploy-race-repro/permissions-reset \
+#     -update -tail -v
+
+cd app
+
+unset MSYS_NO_PATHCONV
+
+trace $CLI bundle deploy
+
+# Declare a permission for a service principal, then re-deploy.
+cat >> databricks.yml <<'YAML'
+
+permissions:
+  - level: CAN_MANAGE
+    service_principal_name: 11111111-2222-3333-4444-555555555555
+YAML
+rm -f out.requests.txt
+$CLI bundle deploy &> LOG.deploy
+# The PUT /api/2.0/permissions/directories/* calls are ApplyWorkspaceRootPermissions
+# replacing the folder ACL with exactly the declared permissions, on every deploy.
+print_requests.py //api/2.0/permissions &>> LOG.perms
+
+trace $CLI bundle destroy --auto-approve &>> LOG.destroy

acceptance/bundle/deploy/deploy-race-repro/permissions-reset/test.toml

@@ -0,0 +1,2 @@
+# Needed for print_requests.py to capture the SetPermissions calls.
+RecordRequests = true

acceptance/bundle/deploy/deploy-race-repro/snapshot-loss/app/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: deploy-race-snapshot-loss
+
+resources:
+  jobs:
+    etl:
+      name: etl
+      tasks:
+        - task_key: run_etl
+          spark_python_task:
+            python_file: ./src/etl.py
+          environment_key: default
+      environments:
+        - environment_key: default
+          spec:
+            client: "2"

acceptance/bundle/deploy/deploy-race-repro/snapshot-loss/app/src/etl.py

@@ -0,0 +1 @@
+print("etl task ran")

acceptance/bundle/deploy/deploy-race-repro/snapshot-loss/script

@@ -0,0 +1,34 @@
+# THROWAWAY investigation harness -- NOT a CI test (disabled via parent test.toml).
+#
+# Behaviour 2: snapshot-loss mass re-upload widening the window.
+#
+# A deploy from a runner with no local sync snapshot re-PUTs every file in one
+# deploy (libs/sync/snapshot.go loadOrNewSnapshot -> empty snapshot). Deleting
+# .databricks and delaying all uploads shows the whole files/ tree is in flight at
+# once, so the delete-before-put window (see ../gap) widens across every file
+# rather than just a single renamed one.
+#
+# Run it by hand to observe the failure:
+#   go test ./acceptance -run TestAccept/bundle/deploy/deploy-race-repro/snapshot-loss \
+#     -update -tail -v
+
+cd app
+
+unset MSYS_NO_PATHCONV
+
+trace $CLI bundle deploy
+trace $CLI bundle run etl
+
+FILES_PATH=$($CLI bundle summary --output json | jq -r '.workspace.file_path')
+
+# Drop the local snapshot so the next deploy re-uploads every file, and delay all
+# uploads so the whole files/ tree is in flight at once. A job running now can race
+# the mass re-upload.
+rm -rf .databricks
+fault.py "POST /api/2.0/workspace-files/import-file${FILES_PATH}/*" 0 0 100 1000
+$CLI bundle deploy &> LOG.deploy &
+DEPLOY_PID=$!
+$CLI bundle run etl &>> LOG.runs || echo "RUN raced the mass re-upload" >> LOG.summary
+wait $DEPLOY_PID
+
+trace $CLI bundle destroy --auto-approve &>> LOG.destroy

acceptance/bundle/deploy/deploy-race-repro/test.toml

@@ -0,0 +1,31 @@
+# THROWAWAY investigation harnesses -- intentionally disabled.
+#
+# Each child test drives a real wall-clock race (a backgrounded bundle run or
+# deploy against an in-flight deploy with a delayed upload), so its output is
+# non-deterministic and it must never run in CI. Run one by hand with -tail -v to
+# observe the behaviour; the committed test ../notebook-not-found-gap proves the
+# core mechanism deterministically.
+#
+# The children split the three DABs deployment behaviours from the investigation:
+#   gap/               delete-before-put gap raced by a live job
+#   snapshot-loss/     snapshot-loss mass re-upload widening the window
+#   permissions-reset/ permission ACL reset on the deployed folder after upload
+#
+# All observed output goes to LOG.* sinks (visible under -v, excluded from the
+# golden diff) so re-running never fails on timing noise.
+Local = false
+Cloud = false
+
+# Bug: bundle deploy is non-atomic (applyDiff deletes before it puts, job repointed
+# only after files.Upload), so a job running during a deploy can transiently fail
+# with "file not found" and self-heal on the next run.
+Badness = "bundle deploy mutates the live files/ tree non-atomically; a concurrent job run can observe a transient not-found"
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [
+    ".databricks",
+    "app/.databricks",
+    "app/databricks.yml",
+    "app/src",
+]

acceptance/bundle/deploy/immutable/script

@@ -16,4 +16,6 @@ trace $CLI jobs get $JOB_ID | jq '.settings.tasks' | jq '.[] | select(.notebook_
 trace $CLI jobs get $JOB_ID | jq '.settings.environments[0].spec.dependencies'
 
 # Redirect run output to a log file — the real workspace produces different output than the local test server.
-$CLI bundle run my_job &> LOG.run
+# The local test server cannot resolve the content-addressed immutable snapshot path, so the run fails locally;
+# tolerate its exit code since the run output is intentionally not asserted.
+errcode $CLI bundle run my_job &> LOG.run

acceptance/bundle/deploy/notebook-not-found-gap/app/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: notebook-gap
+
+resources:
+  jobs:
+    etl:
+      name: etl
+      tasks:
+        - task_key: run_etl
+          spark_python_task:
+            python_file: ./src/etl.py
+          environment_key: default
+      environments:
+        - environment_key: default
+          spec:
+            client: "2"

acceptance/bundle/deploy/notebook-not-found-gap/app/src/etl.py

@@ -0,0 +1 @@
+print("etl task ran")