contest: hw: make sure the machine is healthy before we fetch results

If the machine hung we need to reboot, wait, then fetch.

Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Author: kuba-moo

contest/hw/hwksft.py

@@ -22,7 +22,7 @@
                           kexec_machine, wait_for_results, fetch_results,
                           parse_results, process_crashes, set_log_file,
                           WaitResult, grab_hw_worker_journal, grab_sol_logs,
-                          reboot_machine,
+                          reboot_machine, check_healthy_ssh,
                           CRASH_SENTINEL, _journal_has_crash_sentinel)
 
 # Config:
@@ -263,7 +263,13 @@ def test(binfo, rinfo, cbarg):  # pylint: disable=unused-argument
                 print(f"Max crash retries ({max_crash_retries}) reached, giving up")
                 break
 
-        # 10. Copy back results
+        # 10. Ensure machine is reachable before fetching results
+        if not check_healthy_ssh(machine_ips[0]):
+            print("Machine unreachable, rebooting before fetching results")
+            reboot_machine(config, mc, reservation_id,
+                           machine_ids, machine_ips)
+
+        # 11. Copy back results
         fetch_results(machine_ips, reservation_id, results_path)
 
         # 11. Parse results

contest/hw/lib/deployer.py

@@ -358,6 +358,11 @@ def _journal_has_crash_sentinel(ipaddr):
     return CRASH_SENTINEL in journal
 
 
+def check_healthy_ssh(ipaddr):
+    """Check if a machine is reachable via SSH. Returns True if healthy."""
+    return _ssh_retcode(ipaddr, 'true', timeout=10) == 0
+
+
 def reboot_machine(config, mc, reservation_id, machine_ids, machine_ips):
     """Reboot the machine via SSH, falling back to BMC power cycle."""
     primary_ip = machine_ips[0]
@@ -367,8 +372,7 @@ def _refresh():
         mc.reservation_refresh(reservation_id)
 
     # Check if SSH is responsive at all before trying reboot
-    ssh_ok = _ssh_retcode(primary_ip, 'true', timeout=10) == 0
-    if ssh_ok:
+    if check_healthy_ssh(primary_ip):
         print(f"reboot_machine: rebooting {primary_ip} via SSH")
         _ssh(primary_ip, 'reboot', check=False, timeout=5)
         # Wait for the machine to actually go down before checking