Add efficient pod completion wait to hotloop role

To avoid long waits when tempest tests fail.

- Add hotloop_wait_pod_completion module for immediate failure detection
- Replace 60m timeout with smart polling that exits on Success/Failure

Assisted-By: claude-4-sonnet

Author: hjensas

docs/hotloop_lang.md

@@ -92,6 +92,16 @@ Here's a breakdown of the common attributes within a stage:
   are typically `oc wait` commands in the context of OpenShift, ensuring that
   resources are created, become ready, or reach a desired state before the
   pipeline proceeds. Each item in the list is a command-line string.
+- **`wait_pod_completion`**: (Optional) A list of pod completion wait configurations
+  that efficiently wait for a single pod to reach terminal states (Succeeded or Failed).
+  This provides faster failure detection compared to traditional `oc wait` commands
+  with long timeouts. Each item must define:
+  - **`namespace`**: The Kubernetes namespace to search for pods.
+  - **`labels`**: Label selectors to identify the pod to wait for. Must match
+    exactly one pod.
+  - **`timeout`**: (Optional) Maximum time to wait in seconds. Defaults to 3600.
+  - **`poll_interval`**: (Optional) Interval between status checks in seconds.
+    Defaults to 10.
 - **`run_conditions`**: (Optional) A list of conditions that must be met for a
   stage to execute. Strings `False`, `FALSE` and `false` will be evaluated as
   `False`, otherwise the python boolean equivalent of the value.
@@ -235,6 +245,14 @@ customize the manifest content dynamically.
     - >-
       oc wait -n metallb-system pod -l component=speaker --for condition=Ready
       --timeout=300s"
+  wait_pod_completion:
+    - namespace: openstack
+      labels:
+        operator: test-operator
+        service: tempest
+        workflowStep: "0"
+      timeout: 3600
+      poll_interval: 15
 ```
 
 Here, the `manifest` stage applies the YAML file located at

docs/hotstack_scenarios.md

@@ -86,6 +86,11 @@ Role.
       command-line tool) to execute.  These commands poll the cluster until
       a specific condition is met, ensuring that resources are ready before
       the pipeline proceeds.
+    - `wait_pod_completion`:  A list of pod completion wait configurations
+      that efficiently wait for pods to reach terminal states (Succeeded or
+      Failed). This provides faster failure detection compared to traditional
+      `oc wait` commands with long timeouts, particularly useful for test
+      execution pods that may fail early.
 
 ### `manifests/` (Directory)
 

roles/hotloop/README.md

@@ -72,6 +72,15 @@ Schema for a stage item is:
 
 * `wait_conditions` (list) A list of commands to run after applying the
   manifest, i.e `oc wait --for <condition>`
+* `wait_pod_completion` (list) A list of pod completion wait configurations
+  that efficiently wait for a single pod to reach terminal states (Succeeded or Failed).
+  Each item must define:
+  * `namespace`: (string) The Kubernetes namespace to search for pods.
+  * `labels`: (dict) Label selectors to identify the pod to wait for. Must match
+    exactly one pod.
+  * `timeout`: (int, optional) Maximum time to wait in seconds. Defaults to 3600.
+  * `poll_interval`: (int, optional) Interval between status checks in seconds.
+    Defaults to 10.
 * `run_conditions` (list) A list of conditions that must be met for a stage
   to execute. Strings `False`, `FALSE` and `false` will be evaluated as
   `False`, otherwise the python boolean equivalent of the value.
@@ -116,7 +125,7 @@ Schema for a stage item is:
 
 > **_NOTE_**: Stage items are applied the actions in the following order:
 > `command` -> `shell` -> `manifest` -> `j2_manifest` -> `kustomize` ->
-> `wait_conditions` -> `stages`.
+> `wait_conditions` -> `wait_pod_completion` -> `stages`.
 
 Example:
 
@@ -162,6 +171,25 @@ stages:
       - "oc wait -n openstack-operators -l app.kubernetes.io/name=rabbitmq-cluster-operator deployment --for condition=Available --timeout=300s"
       - "oc wait -n openstack-operators -l app.kubernetes.io/instance=webhook-service service --for jsonpath='{.status.loadBalancer}' --timeout=300s"
 
+  - name: Run tempest tests
+    documentation: |
+      Execute comprehensive OpenStack validation tests using the Tempest framework.
+      Wait for pod completion efficiently without long timeouts on failure.
+      The label selectors must uniquely identify a single pod.
+    manifest: tempest-tests.yml
+    wait_conditions:
+      - >-
+        oc wait -n openstack tempests.test.openstack.org tempest-tests
+        --for condition=ServiceConfigReady --timeout=120s
+    wait_pod_completion:
+      - namespace: openstack
+        labels:
+          operator: test-operator
+          service: tempest
+          workflowStep: "0"
+        timeout: 3600
+        poll_interval: 15
+
   - name: Common MetalLB
     manifest: ../common/metallb.yaml
     wait_conditions:

roles/hotloop/library/hotloop_stage_loader.py

@@ -101,6 +101,7 @@
     "shell",
     "stages",
     "wait_conditions",
+    "wait_pod_completion",
 }
 
 ALLOWED_KUSTOMIZE_KEYS = {
@@ -161,6 +162,72 @@ def _validate_run_conditions(conditions):
         )
 
 
+def _validate_wait_pod_completion(wait_pod_completion_list):
+    """Validates the 'wait_pod_completion' parameter.
+
+    This function checks if the 'wait_pod_completion' parameter is a list
+    of properly structured pod completion wait configurations.
+
+    :param wait_pod_completion_list: The 'wait_pod_completion' parameter to validate.
+    """
+    if not isinstance(wait_pod_completion_list, list):
+        raise TypeError(
+            "'wait_pod_completion' must be a list, got {wait_type}".format(
+                wait_type=type(wait_pod_completion_list)
+            )
+        )
+
+    for i, wait_config in enumerate(wait_pod_completion_list):
+        if not isinstance(wait_config, dict):
+            raise TypeError(
+                "wait_pod_completion[{index}] must be a dict, got {config_type}".format(
+                    index=i, config_type=type(wait_config)
+                )
+            )
+
+        # Check required fields
+        required_fields = {"namespace", "labels"}
+        missing_fields = required_fields - wait_config.keys()
+        if missing_fields:
+            raise ValueError(
+                "wait_pod_completion[{index}] missing required fields: {missing}".format(
+                    index=i, missing=missing_fields
+                )
+            )
+
+        # Validate field types
+        if not isinstance(wait_config["namespace"], str):
+            raise TypeError(
+                "wait_pod_completion[{index}].namespace must be a string, got {ns_type}".format(
+                    index=i, ns_type=type(wait_config["namespace"])
+                )
+            )
+
+        if not isinstance(wait_config["labels"], dict):
+            raise TypeError(
+                "wait_pod_completion[{index}].labels must be a dict, got {labels_type}".format(
+                    index=i, labels_type=type(wait_config["labels"])
+                )
+            )
+
+        # Validate optional fields
+        if "timeout" in wait_config and not isinstance(wait_config["timeout"], int):
+            raise TypeError(
+                "wait_pod_completion[{index}].timeout must be an integer, got {timeout_type}".format(
+                    index=i, timeout_type=type(wait_config["timeout"])
+                )
+            )
+
+        if "poll_interval" in wait_config and not isinstance(
+            wait_config["poll_interval"], int
+        ):
+            raise TypeError(
+                "wait_pod_completion[{index}].poll_interval must be an integer, got {poll_type}".format(
+                    index=i, poll_type=type(wait_config["poll_interval"])
+                )
+            )
+
+
 def _validate_kustomize(kustomize_config):
     """Validates the 'kustomize' parameter.
 
@@ -226,6 +293,11 @@ def _validate_stage(stage, nested=False):
     if not isinstance(stage.get("wait_conditions", []), list):
         raise ValueError("Wait conditions must be a list, {stage}".format(stage=stage))
 
+    if not isinstance(stage.get("wait_pod_completion", []), list):
+        raise ValueError(
+            "Wait pod completion must be a list, {stage}".format(stage=stage)
+        )
+
     if nested and "stages" in stage:
         raise ValueError("Nested stages cannot be nested, {stage}".format(stage=stage))
 
@@ -235,6 +307,9 @@ def _validate_stage(stage, nested=False):
     if "kustomize" in stage:
         _validate_kustomize(stage["kustomize"])
 
+    if "wait_pod_completion" in stage:
+        _validate_wait_pod_completion(stage["wait_pod_completion"])
+
 
 def _load_nested(stages):
     """Load and validates nested stages