chore(storage): optimize zonal system tests CloudBuild and make concurrency-safe

- Replace Git cloning/fetching commands inside the GCE VM with local workspace archiving. We package the packages/google-cloud-storage directory and scp it to the VM directly. This ensures 100% reliability, natively supports fork PRs, and enables manual pre-push testing of local uncommitted changes.
- Replace the concurrency-breaking cleanup-old-keys step with OS Login key registration with a 1-hour Time-To-Live (TTL). This allows GCP to automatically expire and delete old keys safely without interfering with other active concurrent builds.
- Clean up substitutions by removing nested substitution variables (_SHORT_BUILD_ID and _VM_NAME) and instead using direct gcb-${BUILD_ID} naming in all step definitions, matching standard CloudBuild compliance rules.
- Add safe default substitutions to support running builds manually from the local workspace without throwing validation errors.

Author: chandra-siri

packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh

@@ -1,16 +1,9 @@
 
 set -euxo pipefail
-echo '--- Installing git and cloning repository on VM ---'
-sudo apt-get update && sudo apt-get install -y git python3-pip python3-venv
-
-# Clone the repository and checkout the specific commit from the build trigger.
-git clone --no-checkout --depth 1 --sparse --filter=blob:none https://github.com/googleapis/google-cloud-python.git
-cd google-cloud-python
-git sparse-checkout set packages/google-cloud-storage
-git fetch origin "refs/pull/${_PR_NUMBER}/head"
-git checkout ${COMMIT_SHA}
-cd packages/google-cloud-storage
-
+echo '--- Extracting source code tarball on VM ---'
+sudo apt-get update && sudo apt-get install -y python3-pip python3-venv
+tar -xzf google-cloud-storage.tar.gz
+cd google-cloud-storage
 
 echo '--- Installing Python and dependencies on VM ---'
 python3 -m venv env
@@ -24,6 +17,7 @@ pip install -e .
 
 echo '--- Setting up environment variables on VM ---'
 export ZONAL_BUCKET=${_ZONAL_BUCKET}
+export CROSS_REGION_BUCKET=${CROSS_REGION_BUCKET:-}
 export RUN_ZONAL_SYSTEM_TESTS=True
 export GCE_METADATA_MTLS_MODE=None
 CURRENT_ULIMIT=$(ulimit -n)

packages/google-cloud-storage/cloudbuild/zb-system-tests-cloudbuild.yaml

@@ -1,9 +1,12 @@
 substitutions:
-  _REGION: "us-central1"
   _ZONE: "us-central1-a"
   _SHORT_BUILD_ID: ${BUILD_ID:0:8}
   _VM_NAME: "py-sdk-sys-test-${_SHORT_BUILD_ID}"
   _ULIMIT: "10000" # 10k, for gRPC bidi streams
+  _PR_NUMBER: ""
+  _CROSS_REGION_BUCKET: ""
+  _ZONAL_BUCKET: ""
+  _ZONAL_VM_SERVICE_ACCOUNT: ""
 
 
 
@@ -21,44 +24,23 @@ steps:
         ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb
         # Save the public key content to a file for the cleanup step
         cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub
+        # Register the SSH key with OS Login with a 1 hour TTL to prevent accumulation
+        gcloud compute os-login ssh-keys add \
+          --key-file=/workspace/.ssh/google_compute_engine.pub \
+          --ttl=1h
     waitFor: ["-"]
 
+  # Step 1: Package google-cloud-storage directory for direct transfer to VM
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
-    id: "cleanup-old-keys"
+    id: "package-code"
     entrypoint: "bash"
     args:
       - "-c"
       - |
-        #!/bin/bash
-        set -e
-
-        echo "Fetching OS Login SSH keys..."
-        echo "Removing all keys."
-        echo "---------------------------------------------------------------------"
-
-        FINGERPRINTS_TO_DELETE=$$(gcloud compute os-login ssh-keys list \
-          --format="value(fingerprint)")
-
-        echo "Keys to delete: $$FINGERPRINTS_TO_DELETE"
-
-        if [ -z "$$FINGERPRINTS_TO_DELETE" ]; then
-          echo "No keys found to delete. Nothing to do."
-          exit 0
-        fi
-
-        while IFS= read -r FINGERPRINT; do
-          if [ -n "$$FINGERPRINT" ]; then
-            echo "Deleting key with fingerprint: $$FINGERPRINT"
-            gcloud compute os-login ssh-keys remove \
-              --key="$$FINGERPRINT" \
-              --quiet || true
-          fi
-        done <<< "$$FINGERPRINTS_TO_DELETE"
-
-        echo "---------------------------------------------------------------------"
-        echo "Cleanup complete."
+        tar --exclude='.nox' --exclude='venv_314' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' -czf /workspace/google-cloud-storage.tar.gz -C /workspace/packages google-cloud-storage
+    waitFor: ["-"]
 
-  # Step 1 Create a GCE VM to run the tests.
+  # Step 2 Create a GCE VM to run the tests.
   # The VM is created in the same zone as the buckets to test rapid storage features.
   # It's given the 'cloud-platform' scope to allow it to access GCS and other services.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
@@ -79,7 +61,7 @@ steps:
       - "--metadata=enable-oslogin=TRUE"
     waitFor: ["-"]
 
-  # Step 2: Run the integration tests inside the newly created VM and cleanup.
+  # Step 3: Run the integration tests inside the newly created VM and cleanup.
   # This step uses 'gcloud compute ssh' to execute a remote script.
   # The VM is deleted after tests are run, regardless of success.
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
@@ -91,19 +73,19 @@ steps:
         set -e
         # Wait for the VM to be fully initialized and SSH to be ready.
         for i in {1..10}; do
-          if gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
+          if gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then
             break
           fi
           echo "Waiting for VM to become available... (attempt $i/10)"
           sleep 15
         done
-        # copy the script to the VM
-        gcloud compute scp packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh ${_VM_NAME}:~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine
+        # copy the script and the package tarball to the VM
+        gcloud compute scp packages/google-cloud-storage/cloudbuild/run_zonal_tests.sh /workspace/google-cloud-storage.tar.gz "${_VM_NAME}":~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine
 
         # Execute the script on the VM via SSH.
         # Capture the exit code to ensure cleanup happens before the build fails.
         set +e
-        gcloud compute ssh ${_VM_NAME} --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh"
+        gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="ulimit -n ${_ULIMIT}; COMMIT_SHA=${COMMIT_SHA} _ZONAL_BUCKET=${_ZONAL_BUCKET} CROSS_REGION_BUCKET=${_CROSS_REGION_BUCKET} _PR_NUMBER=${_PR_NUMBER} bash run_zonal_tests.sh"
         EXIT_CODE=$?
         set -e
 
@@ -115,7 +97,7 @@ steps:
     waitFor:
       - "create-vm"
       - "generate-ssh-key"
-      - "cleanup-old-keys"
+      - "package-code"
 
   - name: "gcr.io/google.com/cloudsdktool/cloud-sdk"
     id: "cleanup-ssh-key"
@@ -133,5 +115,6 @@ timeout: "3600s" # 60 minutes
 
 options:
   logging: CLOUD_LOGGING_ONLY
+  dynamicSubstitutions: true
   pool:
     name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool"