Run heavy test cases on Saturdays only (#1142)

Signed-off-by: Matthias Büchse <matthias.buechse@alasca.cloud>

Author: mbuechse

.zuul.d/config.yaml

@@ -33,6 +33,7 @@
       preset: default
       iaas: true
       kaas: false
+      section: ''
       do_provision: false
       do_cleanup: true
       sonobuoy_tar_gz_url: >
@@ -69,5 +70,6 @@
       preset: kaas
       iaas: false
       kaas: true
+      section: '.auto'  # only do 'heavy' tests on Saturdays
       do_provision: true
       do_cleanup: false

Standards/scs-0003-v1-sovereign-cloud-standards-yaml.md

@@ -197,7 +197,6 @@ The following fields are valid for every script descriptor:
 
 | Key               | Type   | Description                                                                                    | Example    |
 | ----------------- | ------ | ---------------------------------------------------------------------------------------------- | ---------- |
-| `section`         | String | _Optional_ what section to associate this check with (sections can be checked in isolation)    | `weekly`   |
 | `testcases`       | Array  | List of all test cases; each entry being a test-case descriptor                                | (see below)  |
 
 Additional fields are valid depending on whether the check is automated or manual.
@@ -226,6 +225,7 @@ TBD
 | ----------------- | --------------- | ------------------------------------------------------------------------------------------------- | ----------------- |
 | `id`              | String          | Identifier for this test case (immutable and unique within this module)                           | `image-md-check`  |
 | `lifetime`        | String          | One of: `day`, `week` (_default_), `month`, `quarter`, `year`                                     | `day`             |
+| `section`         | String          | _Optional_ what section to associate this testcase with (sections can be checked in isolation)    | `weekly`          |
 | `description`     | String          | Short description of the test case (markdown allowed, but keep it short for CLI users)            |                   |
 | `url`             | String          | URL pointing to the relevant SCS documentation for the testcase                                   |                   |
 

Tests/scs-compatible-kaas.yaml

@@ -17,24 +17,29 @@ scripts:
   testcases:
   - id: cncf-k8s-conformance
     lifetime: year
+    section: heavy
     description: Must fulfill all requirements of _CNCF Kubernetes conformance_.
     url: https://github.com/cncf/k8s-conformance/tree/master
 - executable: ./kaas/k8s-version-policy/k8s_version_policy.py
   args: -k {kubeconfig}
   testcases:
   - id: version-policy-check
+    section: light
     description: Must fulfill all requirements of scs-0210-v2.
     url: https://docs.scs.community/standards/scs-0210-v2-k8s-version-policy#decision
 - executable: ./kaas/k8s-node-distribution/k8s_node_distribution_check.py
   args: -k {kubeconfig}
   testcases:
   - id: node-distribution-check
+    section: light
     description: Must fulfill all requirements of scs-0214-v2.
     url: https://docs.scs.community/standards/scs-0214-v2-k8s-node-distribution#decision
 - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py
   args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-0219 -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"' --execution-mode {execution_mode}
   testcases:
   - id: kaas-networking-check
+    lifetime: month
+    section: heavy
     description: Must fulfill all requirements of scs-0219-v1.
     url: https://docs.scs.community/standards/scs-0219-v1-kaas-networking#decision
 modules:

Tests/scs-compliance-check.py

@@ -339,7 +339,7 @@ def main(argv):
         script = tc_script_lookup[tc_id]
         if 'executable' not in script:
             continue  # manual check
-        if config.sections and script.get('section') not in config.sections:
+        if config.sections and testcase_lookup.get(tc_id, {}).get('section') not in config.sections:
             continue
         if config.tests and not config.tests.match(tc_id):
             continue

Tests/scs-test-runner.py

@@ -5,6 +5,7 @@
 #
 # (c) Matthias Büchse <matthias.buechse@cloudandheat.com>
 # SPDX-License-Identifier: Apache-2.0
+from datetime import date
 import logging
 import os
 import os.path
@@ -68,12 +69,14 @@ def get_subject_mapping(self, subject):
     def abspath(self, path):
         return os.path.join(self.cwd, path)
 
-    def build_check_command(self, scope, subject, output):
+    def build_check_command(self, scope, subject, sections, output):
         # TODO figure out when to supply --debug here (but keep separated from our --debug)
         args = [
             sys.executable, self.scs_compliance_check, self.abspath(self.scopes[scope]['spec']),
             '--debug', '-C', '-o', output, '-s', subject,
         ]
+        if sections:
+            args.extend(['--sections', sections])
         for key, value in self.get_subject_mapping(subject).items():
             args.extend(['-a', f'{key}={value}'])
         return {'args': args}
@@ -173,12 +176,13 @@ def _move_file(source_path, target_path):
 @cli.command()
 @click.option('--scope', 'scopes', type=str)
 @click.option('--subject', 'subjects', type=str)
+@click.option('--section', 'sections', type=str)
 @click.option('--preset', 'preset', type=str)
 @click.option('--num-workers', 'num_workers', type=int, default=5)
 @click.option('--monitor-url', 'monitor_url', type=str, default=MONITOR_URL)
 @click.option('-o', '--output', 'report_yaml', type=click.Path(exists=False), default=None)
 @click.pass_obj
-def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml):
+def run(cfg, scopes, subjects, sections, preset, num_workers, monitor_url, report_yaml):
     """
     run compliance tests and upload results to compliance monitor
     """
@@ -196,13 +200,23 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml):
         subjects = [subject.strip() for subject in subjects.split(',')] if subjects else []
     if not scopes or not subjects:
         raise click.UsageError('both scope(s) and subject(s) must be non-empty')
+    if sections == '.auto':
+        today = date.today()
+        # https://docs.python.org/3/library/datetime.html#datetime.date.weekday
+        # Return the day of the week as an integer, where Monday is 0 and Sunday is 6.
+        weekday = today.weekday()
+        if weekday == 5:  # Saturday
+            sections = 'light,medium,heavy'
+        else:
+            sections = 'light,medium'
+        logger.info(f'auto-selected sections: {sections}')
     logger.debug(f'running tests for scope(s) {", ".join(scopes)} and subject(s) {", ".join(subjects)}')
     logger.debug(f'monitor url: {monitor_url}, num_workers: {num_workers}, output: {report_yaml}')
     with tempfile.TemporaryDirectory(dir=cfg.cwd) as tdirname:
         report_yaml_tmp = os.path.join(tdirname, 'report.yaml')
         jobs = [(scope, subject) for scope in scopes for subject in subjects]
         outputs = [os.path.join(tdirname, f'report-{idx}.yaml') for idx in range(len(jobs))]
-        commands = [cfg.build_check_command(job[0], job[1], output) for job, output in zip(jobs, outputs)]
+        commands = [cfg.build_check_command(job[0], job[1], sections, output) for job, output in zip(jobs, outputs)]
         _run_commands(commands, num_workers=num_workers)
         _concat_files(outputs, report_yaml_tmp)
         subprocess.run(**cfg.build_sign_command(report_yaml_tmp))

Tests/scs_cert_lib.py

@@ -16,10 +16,10 @@
 # valid keywords for various parts of the spec, to be checked using `check_keywords`
 KEYWORDS = {
     'spec': ('uuid', 'name', 'url', 'versions', 'prerequisite', 'variables', 'scripts', 'modules', 'timeline'),
-    'scripts': ('executable', 'env', 'args', 'section', 'testcases'),
+    'scripts': ('executable', 'env', 'args', 'testcases'),
     'versions': ('version', 'include', 'targets', 'stabilized_at'),
     'modules': ('id', 'targets', 'url', 'name', 'parameters'),
-    'testcases': ('lifetime', 'id', 'description', 'url'),
+    'testcases': ('lifetime', 'section', 'id', 'description', 'url'),
     'include': ('ref', 'parameters'),
 }
 # The canonical result values are -1, 0, and 1, for FAIL, ABORT, and PASS, respectively;

playbooks/compliance_check.yaml

@@ -8,7 +8,7 @@
       ansible.builtin.command:
         cmd: >
           python3 Tests/scs-test-runner.py --config Tests/config.toml --debug
-          run --preset {{ preset }}
+          run --preset {{ preset }} --section '{{ section }}'
           --output "{{ ansible_user_dir }}/zuul-output/artifacts/report.yaml"
         chdir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
       changed_when: true