Allow some Sonobuoy tests (KaaS) to fail.

Signed-off-by: Thomas Güttler <thomas.guettler@syself.com>

Author: guettli

Tests/kaas/sonobuoy_handler/sonobuoy_handler.py

@@ -1,7 +1,8 @@
 from collections import Counter
-import json
 import logging
+import re
 import os
+import sys
 import os.path
 import shlex
 import shutil
@@ -14,18 +15,20 @@
 
 def _find_sonobuoy():
     """find sonobuoy in PATH, but also in a fixed location at ~/.local/bin to simplify use with Ansible"""
-    result = shutil.which('sonobuoy')
+    result = shutil.which("sonobuoy")
     if result:
         return result
-    logger.debug('sonobuoy not in PATH, trying $HOME/.local/bin')
-    result = os.path.join(os.path.expanduser('~'), '.local', 'bin', 'sonobuoy')
+    logger.debug("sonobuoy not in PATH, trying $HOME/.local/bin")
+    result = os.path.join(os.path.expanduser("~"), ".local", "bin", "sonobuoy")
     if os.path.exists(result):
         return result
-    logger.debug('sonobuoy executable not found; expect errors')
+    logger.debug("sonobuoy executable not found; expect errors")
 
 
 def _fmt_result(counter):
-    return ', '.join(f"{counter.get(key, 0)} {key}" for key in ('passed', 'failed', 'skipped'))
+    return ", ".join(
+        f"{counter.get(key, 0)} {key}" for key in ("passed", "failed", "skipped")
+    )
 
 
 class SonobuoyHandler:
@@ -69,20 +72,10 @@ def _sonobuoy_run(self):
     def _sonobuoy_delete(self):
         self._invoke_sonobuoy("delete", "--wait")
 
-    def _sonobuoy_status_result(self):
-        process = self._invoke_sonobuoy("status", "--json", capture_output=True)
-        json_data = json.loads(process.stdout)
-        counter = Counter()
-        for entry in json_data["plugins"]:
-            logger.debug(f"plugin {entry['plugin']}: {_fmt_result(entry['result-counts'])}")
-            for key, value in entry["result-counts"].items():
-                counter[key] += value
-        return counter
-
     def _eval_result(self, counter):
         """evaluate test results and return return code"""
         result_message = f"sonobuoy reports {_fmt_result(counter)}"
-        if counter['failed']:
+        if counter["failed"]:
             logger.error(result_message)
             return 3
         logger.info(result_message)
@@ -95,34 +88,21 @@ def _preflight_check(self):
         if not self.sonobuoy:
             raise RuntimeError("sonobuoy executable not found; is it in PATH?")
 
-    def _sonobuoy_retrieve_result(self, plugin='e2e'):
+    def _sonobuoy_retrieve_result(self, plugin="e2e"):
         """
         Invoke sonobuoy to retrieve results and to store them in a subdirectory of
         the working directory. Analyze the results yaml file for given `plugin` and
-        log each failure as ERROR. Return summary dict like `_sonobuoy_status_result`.
+        log each failure as ERROR. Return summary dict.
         """
         logger.debug(f"retrieving results to {self.result_dir_name}")
         result_dir = os.path.join(self.working_directory, self.result_dir_name)
         os.makedirs(result_dir, exist_ok=True)
 
         self._invoke_sonobuoy("retrieve", "-x", result_dir)
-        yaml_path = os.path.join(result_dir, 'plugins', plugin, 'sonobuoy_results.yaml')
+        yaml_path = os.path.join(result_dir, "plugins", plugin, "sonobuoy_results.yaml")
         logger.debug(f"parsing results from {yaml_path}")
-        with open(yaml_path, "r") as fileobj:
-            result_obj = yaml.load(fileobj.read(), yaml.SafeLoader)
-        counter = Counter()
-        for item1 in result_obj.get('items', ()):
-            # file ...
-            for item2 in item1.get('items', ()):
-                # suite ...
-                for item in item2.get('items', ()):
-                    # testcase ... or so
-                    status = item.get('status', 'skipped')
-                    counter[status] += 1
-                    if status == 'failed':
-                        logger.error(f"FAILED: {item['name']}")  # <-- this is why this method exists!
-        logger.info(f"{plugin} results: {_fmt_result(counter)}")
-        return counter
+
+        return sonobuoy_parse_result(plugin, yaml_path)
 
     def run(self):
         """
@@ -132,16 +112,106 @@ def run(self):
         self._preflight_check()
         try:
             self._sonobuoy_run()
-            return_code = self._eval_result(self._sonobuoy_status_result())
+            counter = self._sonobuoy_retrieve_result()
+            return_code = self._eval_result(counter)
             print(self.check_name + ": " + ("PASS", "FAIL")[min(1, return_code)])
-            try:
-                self._sonobuoy_retrieve_result()
-            except Exception:
-                # swallow exception for the time being
-                logger.debug('problem retrieving results', exc_info=True)
             return return_code
         except BaseException:
             logger.exception("something went wrong")
             return 112
         finally:
             self._sonobuoy_delete()
+
+
+def sonobuoy_parse_result(plugin, sonobuoy_results_yaml_path):
+    with open(sonobuoy_results_yaml_path, "r") as fileobj:
+        result_obj = yaml.load(fileobj.read(), yaml.SafeLoader)
+    counter = Counter()
+    for item1 in result_obj.get("items", ()):
+        # file ...
+        for item2 in item1.get("items", ()):
+            # suite ...
+            for item in item2.get("items", ()):
+                # testcase ... or so
+                status = item.get("status", "skipped")
+                if status == "failed":
+                    if ok_to_fail(item["name"]):
+                        status = "failed_ok"
+                    else:
+                        logger.error(f"FAILED: {item['name']}")
+                counter[status] += 1
+
+    logger.info(f"{plugin} results: {_fmt_result(counter)}")
+    return counter
+
+
+def ok_to_fail(test_name):
+    # TODO: Make this configurable
+    regexs_and_reason = [
+        # InternalIP still used in tests. Having only an ExternalIP is considered valid by SCS:
+        (
+            "HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol",
+            "Fails when a cluster has no InternalIP (only ExternalIP): https://github.com/kubernetes/kubernetes/issues/136626",
+        ),
+        (
+            "Services should be able to switch session affinity for NodePort service",
+            "Fails when a cluster has no InternalIP (only ExternalIP): https://github.com/kubernetes/kubernetes/issues/136626",
+        ),
+        (
+            "Services should have session affinity work for NodePort service",
+            "Fails when a cluster has no InternalIP (only ExternalIP): https://github.com/kubernetes/kubernetes/issues/136626",
+        ),
+        (
+            "validates that there exists conflict between pods with same hostPort and protocol but one using 0.0.0.0 hostIP",
+            "Fails when a cluster has no InternalIP (only ExternalIP): https://github.com/kubernetes/kubernetes/issues/136626",
+        ),
+        # Was flaky
+        (
+            "Netpol NetworkPolicy between server and client should allow ingress access from updated namespace",
+            "Flaky test. Fix in v1.36 (https://github.com/kubernetes/kubernetes/pull/136715)",
+        ),
+        # SCTP is optional
+        (
+            "Feature:SCTPConnectivity",
+            "SCTPConnectivity is optional. Currently Cilium does not support it by default.",
+        ),
+        # Tests skipped by Cilium: https://github.com/cilium/cilium/blob/main/.github/workflows/k8s-kind-network-policies-e2e.yaml#L177-L185
+        (
+            "should.allow.egress.access.to.server.in.CIDR.block",
+            "https://github.com/cilium/cilium/issues/9209",
+        ),
+        (
+            "should.ensure.an.IP.overlapping.both.IPBlock.CIDR.and.IPBlock.Except.is.allowed",
+            "https://github.com/cilium/cilium/issues/9209",
+        ),
+        (
+            "should.enforce.except.clause.while.egress.access.to.server.in.CIDR.block",
+            "https://github.com/cilium/cilium/issues/9209",
+        ),
+    ]
+    name = test_name
+    for regex, _reason in regexs_and_reason:
+        if re.search(regex, name):
+            return True
+    return False
+
+
+def usage():
+    print(
+        f"""{sys.argv[0]} path-to/sonobuoy_results.yaml
+Read sonobuoy_results.yaml and print the results.
+"""
+    )
+    sys.exit(3)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        usage()
+    sonobuoy_results_yaml_path = sys.argv[1]
+    if not os.path.exists(sonobuoy_results_yaml_path):
+        print(f"{sonobuoy_results_yaml_path} does not exist")
+        usage()
+    counter = sonobuoy_parse_result("", sys.argv[1])
+    for key, value in counter.items():
+        print(f"{key}: {value}")