Restructure CM details page

resolves #1055

Signed-off-by: Matthias Büchse <matthias.buechse@alasca.cloud>

Author: mbuechse

Tests/scs_cert_lib.py

@@ -64,6 +64,7 @@ def _resolve_spec(spec: dict):
             id_ = testcase['id']
             if id_ in testcase_lookup:
                 raise RuntimeError(f"duplicate testcase {id_}")
+            testcase['attn'] = 0  # count: how many versions list this in target 'main'?
             testcase_lookup[id_] = testcase
             tc_script_lookup[id_] = script
     module_lookup = {module['id']: module for module in spec['modules']}
@@ -92,7 +93,14 @@ def _resolve_spec(spec: dict):
         for inc in version['include']:
             for target, tc_ids in inc['module'].get('targets', {}).items():
                 targets[target].update(tc_ids)
+        tc_target = {}
+        for target, tc_ids in targets.items():
+            for tc_id in tc_ids:
+                tc_target[tc_id] = target
+        for tc_id in targets.get('main', ()):
+            testcase_lookup[tc_id]['attn'] += 1
         version['targets'] = {target: sorted(tc_ids) for target, tc_ids in targets.items()}
+        version['tc_target'] = tc_target
     # step 4b. resolve references to versions in timeline
     # on second thought, let's not go there: it's a canonical extension map, and it should remain that way.
     # however, we still have to look for name errors

compliance-monitor/monitor.py

@@ -49,14 +49,14 @@
 
 
 try:
-    from scs_cert_lib import load_spec, annotate_validity, add_period, evaluate
+    from scs_cert_lib import load_spec, annotate_validity, add_period, eval_buckets, evaluate
 except ImportError:
     # the following course of action is not unproblematic because the Tests directory will be
     # mounted to the Docker instance, hence it's hard to tell what version we are gonna get;
     # however, unlike the reloading of the config, the import only happens once, and at that point
     # in time, both monitor.py and scs_cert_lib.py should come from the same git checkout
     import sys; sys.path.insert(0, os.path.abspath('../Tests'))  # noqa: E702
-    from scs_cert_lib import load_spec, annotate_validity, add_period, evaluate
+    from scs_cert_lib import load_spec, annotate_validity, add_period, eval_buckets, evaluate
 
 
 class Settings:
@@ -251,6 +251,7 @@ def _evaluate_version(version, scope_results):
     return {
         'result': target_results['main']['result'],
         'targets': target_results,
+        'tc_target': version['tc_target'],
         'validity': version['validity'],
     }
 
@@ -282,6 +283,11 @@ def _evaluate_scope(spec, scope_results, include_drafts=False):
         'name': spec['name'],
         'testcases': testcases,
         'results': scope_results,
+        'buckets': {
+            # sort testcase that occur any main target on top of those that don't
+            res: sorted(tc_ids, key=lambda tc_id: (not testcases[tc_id]['attn'], tc_id))
+            for res, tc_ids in eval_buckets(scope_results, testcases).items()
+        },
         'versions': version_results,
         'relevant': relevant,
         'passed': passed,
@@ -724,7 +730,7 @@ async def get_scope(
     view_type: ViewType,
     scopeuuid: str,
 ):
-    spec = get_scopes()[scopeuuid].spec
+    spec = get_scopes()[scopeuuid]
     versions = spec['versions']
     # sort by name, and all drafts after all non-drafts
     column_data = [

compliance-monitor/templates/details.md.j2

@@ -1,3 +1,7 @@
+Note: testcase occurrences inside scope versions are marked
+bold "**X**" for required testcases (target _main_) and
+plain "x" for recommended testcases (with the specific target in the tooltip).
+
 {% if results | length > 1 %}
 Jump to
 
@@ -20,30 +24,31 @@ Jump to
 
 No recent test results available.
 
-{% endif -%}
-{% for version in scope_result.relevant -%}
-{%- set version_result = scope_result.versions[version] -%}
-### {{ version }} ({{ version_result.validity }}): {{ version_result.result | verdict }}
-{% for target, target_result in version_result.targets.items() -%}
-#### Target {{ target }}: {{ target_result.result | verdict }}
-
-| testcase id | result | description |
-|---|---|---|
-{% for testcase_id in target_result.testcases -%}
-{% set testcase = scope_result.testcases[testcase_id] -%}
+{% else %}
+
+{% for cat, res in (('FAILED', -1), ('Missing', 0), ('Passed', 1)) -%}
+{% set bucket = scope_result.buckets[res] -%}
+{% if bucket -%}
+### {{cat}} testcases
+
+| testcase id |{% for version in scope_result.relevant %} {{ version }} |{% endfor %} description |
+|---|{% for version in scope_result.relevant %}---|{% endfor %}---|
+{% for testcase_id in bucket -%}
 {% set res = scope_result.results[testcase_id] if testcase_id in scope_result.results else dict(result=0) -%}
-| {% if res.result != 1 %}⚠️ {% endif %}{{ testcase.id }} |
-{#- #} {% if res.report -%}
-[{{ res.result | verdict_check }}]({{ report_url(res.report, version, testcase_id) }})
+{#- #}  {% if res.report -%}
+[{{ testcase_id }}]({{ report_url(res.report, version, testcase_id) }})
 {%- else -%}
-{{ res.result | verdict_check }}
+{{ testcase_id }}
 {%- endif -%}
 {% if res.report %} { title="{{ res.report }} ({{ res.checked_at }})" }{% endif %} {# -#}
-| {{ testcase.description | trim }} |
-{% endfor %}
-{% endfor -%}
-{% endfor -%}
-{% endfor -%}
-{% endfor -%}
+{% for version in scope_result.relevant %}| {%
+set tgt = scope_result.versions[version].tc_target[testcase_id]
+%}{% if tgt == 'main' %}**X** { title="main" }{% elif tgt %}x { title="{{ tgt }}" }{% endif %} {% endfor -%}
+| {{ scope_result.testcases[testcase_id].description | trim }} |
+{% endfor %}{# testcase #}
+{% endif %}{# bucket nonempty -#}
+{% endfor %}{# categories #}
 
-{#{results}#}
+{% endif %}{# results available -#}
+{% endfor %}{# scope -#}
+{% endfor %}{# subject -#}