contest: hw: parse nested results and apply guess indicators

Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Author: kuba-moo

contest/hw/lib/deployer.py

@@ -12,7 +12,8 @@
 import time
 from dataclasses import dataclass, field
 
-from lib.nipa import has_crash, extract_crash
+from lib.nipa import (has_crash, extract_crash, guess_indicators,
+                      result_from_indicators, parse_nested_tests, namify)
 
 
 # Log file handle, set by set_log_file() before builds start.
@@ -416,17 +417,6 @@ def fetch_results(machine_ips, reservation_id, results_path):
               check=False)
 
 
-def _retcode_to_result(retcode, stdout):
-    """Map a test return code + stdout to pass/fail/skip."""
-    if retcode == 4:
-        return 'skip'
-    if retcode != 0:
-        return 'fail'
-    if 'ok' not in stdout.lower():
-        return 'skip'
-    return 'pass'
-
-
 def parse_results(results_path, link):
     """Parse fetched test output into a vmksft-p-style result list.
 
@@ -464,28 +454,36 @@ def parse_results(results_path, link):
                 with open(stdout_path, encoding='utf-8') as fp:
                     stdout = fp.read()
 
-            # Determine result
-            result = _retcode_to_result(retcode, stdout)
+            # Determine result using indicators
+            indicators = guess_indicators(stdout)
+            result = result_from_indicators(retcode, indicators)
+
+            # Parse nested subtests
+            nested = parse_nested_tests(stdout, namify)
 
             # Determine retry result if present
             retry_result = None
+            retry_nested = None
             if 'retry_retcode' in info:
                 retry_stdout = ''
                 retry_dir = os.path.join(output_dir, entry + '-retry')
                 retry_stdout_path = os.path.join(retry_dir, 'stdout')
                 if os.path.exists(retry_stdout_path):
                     with open(retry_stdout_path, encoding='utf-8') as fp:
                         retry_stdout = fp.read()
-                retry_result = _retcode_to_result(info['retry_retcode'],
-                                                  retry_stdout)
+                retry_indicators = guess_indicators(retry_stdout)
+                retry_result = result_from_indicators(info['retry_retcode'],
+                                                      retry_indicators)
+                if nested:
+                    retry_nested = list(nested)
+                    parse_nested_tests(retry_stdout, namify,
+                                       prev_results=retry_nested)
 
-            safe_name = re.sub(r'[^0-9a-zA-Z]+', '-', prog)
-            if safe_name and safe_name[-1] == '-':
-                safe_name = safe_name[:-1]
+            safe_name = namify(prog)
 
             outcome = {
                 'test': safe_name or entry,
-                'group': f'selftests-{re.sub(r"[^0-9a-zA-Z]+", "-", target).rstrip("-")}',
+                'group': f'selftests-{namify(target)}',
                 'result': result,
                 'link': link,
             }
@@ -495,6 +493,10 @@ def parse_results(results_path, link):
                 outcome['retry'] = retry_result
             if info.get('crashes'):
                 outcome['crashes'] = info['crashes']
+            if retry_nested:
+                outcome['results'] = retry_nested
+            elif nested:
+                outcome['results'] = nested
             cases.append(outcome)
 
     # Check .attempted for crashed tests (attempted but no output)

contest/hw/lib/nipa.py

@@ -19,6 +19,10 @@
 from contest.remote.lib.crash import has_crash  # noqa: E402, F401
 from contest.remote.lib.crash import extract_crash  # noqa: E402, F401
 from contest.remote.lib.crash import crash_finger_print  # noqa: E402, F401
+from contest.remote.lib.results import guess_indicators  # noqa: E402, F401
+from contest.remote.lib.results import result_from_indicators  # noqa: E402, F401
+from contest.remote.lib.results import parse_nested_tests  # noqa: E402, F401
 from contest.remote.lib.cbarg import CbArg  # noqa: E402, F401
 from contest.remote.lib.fetcher import Fetcher  # noqa: E402, F401
+from contest.remote.lib.fetcher import namify  # noqa: E402, F401
 from core import NipaLifetime  # noqa: E402, F401

contest/hw/lib/runner.py

@@ -4,11 +4,10 @@
 
 import json
 import os
-import re
 import subprocess
 import time
 
-from lib.nipa import has_crash, extract_crash
+from lib.nipa import has_crash, extract_crash, namify
 
 
 def find_newest_unseen(tests_dir):
@@ -125,15 +124,6 @@ def _list_tests(test_dir):
     return tests
 
 
-def _namify(what):
-    """Convert test name to a safe identifier."""
-    if not what:
-        return "no-name"
-    name = re.sub(r'[^0-9a-zA-Z]+', '-', what)
-    if name and name[-1] == '-':
-        name = name[:-1]
-    return name
-
 
 def load_filters(test_dir):
     """Load crash filters from filters.json in the test directory.
@@ -251,7 +241,7 @@ def run_tests(test_dir, results_dir):
 
     for test_idx, (target, prog) in enumerate(tests):
         test_name = f"{target}:{prog}"
-        safe_name = _namify(prog)
+        safe_name = namify(prog)
         dir_name = f"{test_idx}-{safe_name}"
 
         if test_name in previously_attempted:

contest/hw/tests/test_hw_worker.py

@@ -11,7 +11,8 @@
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
 
 from lib.runner import (find_newest_unseen, mark_all_seen, load_attempted,
-                        mark_attempted, run_tests, DmesgReader, _namify)
+                        mark_attempted, run_tests, DmesgReader)
+from lib.nipa import namify
 
 
 class TestFindNewestUnseen(unittest.TestCase):
@@ -258,19 +259,19 @@ def test_load_attempted_empty_file(self):
 
 class TestNamify(unittest.TestCase):
     def test_simple(self):
-        self.assertEqual(_namify('test_name'), 'test-name')
+        self.assertEqual(namify('test_name'), 'test-name')
 
     def test_special_chars(self):
-        self.assertEqual(_namify('test/name.sh'), 'test-name-sh')
+        self.assertEqual(namify('test/name.sh'), 'test-name-sh')
 
     def test_trailing_dash(self):
-        self.assertEqual(_namify('test/'), 'test')
+        self.assertEqual(namify('test/'), 'test')
 
     def test_empty(self):
-        self.assertEqual(_namify(''), 'no-name')
+        self.assertEqual(namify(''), 'no-name')
 
     def test_none(self):
-        self.assertEqual(_namify(None), 'no-name')
+        self.assertEqual(namify(None), 'no-name')
 
 
 class TestRunTests(unittest.TestCase):

contest/hw/tests/test_hwksft.py

@@ -263,8 +263,12 @@ def test_parse_results(self):
             output_base = os.path.join(tmpdir, 'test-outputs')
 
             for idx, (target, prog, rc, stdout_text) in enumerate([
-                ('net', 'test1.sh', 0, 'ok 1 test1\n'),
-                ('net', 'test2.sh', 1, 'not ok 1 test2\n'),
+                ('net', 'test1.sh', 0,
+                 'TAP version 13\n1..1\n'
+                 'ok 1 selftests: net: test1.sh\n'),
+                ('net', 'test2.sh', 1,
+                 'TAP version 13\n1..1\n'
+                 'not ok 1 selftests: net: test2.sh\n'),
             ]):
                 test_dir = os.path.join(output_base, f'{idx}-{prog.replace(".", "-")}')
                 os.makedirs(test_dir)

contest/remote/lib/__init__.py

@@ -2,6 +2,7 @@
 
 from .fetcher import Fetcher, namify
 from .loadavg import wait_loadavg
-from .vm import VM, new_vm, guess_indicators
+from .vm import VM, new_vm
 from .cbarg import CbArg
 from .crash import has_crash, extract_crash
+from .results import guess_indicators, result_from_indicators, parse_nested_tests

contest/remote/lib/results.py

@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: GPL-2.0
+
+"""Shared result parsing helpers for kselftest output.
+
+Functions in this module are used by both vmksft-p (VM testing) and
+hwksft (HW testing) to interpret test output.
+"""
+
+import re
+
+
+def guess_indicators(output):
+    """Scan test output for pass/fail/skip indicator strings.
+
+    Returns a dict with 'pass', 'fail', 'skip' boolean values.
+    """
+    return {
+        "fail": output.find("[FAIL]") != -1 or output.find("[fail]") != -1 or
+                output.find(" FAIL:") != -1 or
+                output.find("\nnot ok 1 selftests: ") != -1 or
+                output.find("\n# not ok 1") != -1,
+        "skip": output.find("[SKIP]") != -1 or output.find("[skip]") != -1 or
+                output.find(" # SKIP") != -1 or output.find("SKIP:") != -1,
+        "pass": output.find("[OKAY]") != -1 or output.find("[PASS]") != -1 or
+                output.find("[ OK ]") != -1 or output.find("[OK]") != -1 or
+                output.find("[ ok ]") != -1 or output.find("[pass]") != -1 or
+                output.find("PASSED all ") != -1 or
+                output.find("\nok 1 selftests: ") != -1 or
+                bool(re.search(
+                    r"# Totals: pass:[1-9]\d* fail:0 (xfail:0 )?(xpass:0 )?skip:0 error:0",
+                    output)),
+    }
+
+
+def result_from_indicators(retcode, indicators):
+    """Determine test result from return code and output indicators."""
+    result = 'pass'
+    if indicators["skip"] or not indicators["pass"]:
+        result = 'skip'
+    if retcode == 4:
+        result = 'skip'
+    elif retcode:
+        result = 'fail'
+    if indicators["fail"]:
+        result = 'fail'
+    return result
+
+
+def parse_nested_tests(full_run, namify_fn, prev_results=None):
+    """Parse nested KTAP subtests from test output.
+
+    Args:
+        full_run: full stdout of the test run
+        namify_fn: function to sanitize test names
+        prev_results: if not None, this is a retry run — merge results
+                      into prev_results instead of creating new entries
+
+    Returns a list of subtest dicts (empty if prev_results is used,
+    since results are merged in-place).
+    """
+    tests = []
+    nested_tests = False
+
+    result_re = re.compile(
+        r"(not )?ok (\d+)( -)? ([^#]*[^ ])( +# +)?([^ ].*)?$")
+    time_re = re.compile(r"time=(\d+)ms")
+
+    for line in full_run.split('\n'):
+        # nested subtests support: we parse the comments from 'TAP version'
+        if nested_tests:
+            if line.startswith("# "):
+                line = line[2:]
+            else:
+                nested_tests = False
+        elif line.startswith("# TAP version "):
+            nested_tests = True
+            continue
+
+        if not nested_tests:
+            continue
+
+        if line.startswith("ok "):
+            result = "pass"
+        elif line.startswith("not ok "):
+            result = "fail"
+        else:
+            continue
+
+        v = result_re.match(line).groups()
+        r = {'test': namify_fn(v[3])}
+
+        if len(v) > 5 and v[4] and v[5]:
+            if v[5].lower().startswith('skip'):
+                result = "skip"
+
+            t = time_re.findall(v[5].lower())
+            if t:
+                r['time'] = round(int(t[-1]) / 1000.)  # take the last one
+
+        r['result'] = result
+
+        if prev_results is not None:
+            for entry in prev_results:
+                if entry['test'] == r['test']:
+                    entry['retry'] = result
+                    break
+            else:
+                # the first run didn't validate this test: add it to the list
+                r['result'] = 'skip'
+                r['retry'] = result
+                prev_results.append(r)
+        else:
+            tests.append(r)
+
+    # return an empty list when there are prev results: no replacement needed
+    return tests

contest/remote/lib/vm.py

@@ -7,7 +7,6 @@
 import json
 import os
 import psutil
-import re
 import select
 import shutil
 import signal
@@ -469,18 +468,3 @@ def new_vm(results_path, vm_id, thr=None, vm=None, config=None, cwd=None):
             vm.dump_log(results_path + f'/vm-crashed-{thr_pfx}{vm_id}-{i}')
             vm.stop()
 
-
-def guess_indicators(output):
-    return {
-        "fail": output.find("[FAIL]") != -1 or output.find("[fail]") != -1 or \
-                output.find(" FAIL:") != -1 or \
-                output.find("\nnot ok 1 selftests: ") != -1 or \
-                output.find("\n# not ok 1") != -1,
-        "skip": output.find("[SKIP]") != -1 or output.find("[skip]") != -1 or \
-                output.find(" # SKIP") != -1 or output.find("SKIP:") != -1,
-        "pass": output.find("[OKAY]") != -1 or output.find("[PASS]") != -1 or \
-                output.find("[ OK ]") != -1 or output.find("[OK]") != -1 or \
-                output.find("[ ok ]") != -1 or output.find("[pass]") != -1 or \
-                output.find("PASSED all ") != -1 or output.find("\nok 1 selftests: ") != -1 or \
-                bool(re.search(r"# Totals: pass:[1-9]\d* fail:0 (xfail:0 )?(xpass:0 )?skip:0 error:0", output)),
-    }