[env_op_images] Add CRI-O pull verification to pulled-images report

nemarjan · nemarjan · commit b5d929461e33 · 2026-04-17T23:47:03.000+01:00
Cross-reference the pulled-images report with CRI-O journal logs
from cluster nodes to confirm which images were actually pulled by
the container runtime. Runs automatically when kubeconfig is defined,
same as the pulled-images report itself.

Co-authored-by: Cursor &lt;cursor@cursor.com&gt;

Signed-off-by: nemarjan &lt;nemarjan@redhat.com&gt;
diff --git a/ci/playbooks/collect-logs.yml b/ci/playbooks/collect-logs.yml
@@ -55,7 +55,7 @@
                 chdir: "{{ ansible_user_dir }}/zuul-output/logs/ci-framework-data"
                 cmd: |
                   cp -ra {{ ansible_user_dir }}/ci-framework-data/logs . ;
-                  cp -ra {{ ansible_user_dir }}/ci-framework-data/artifacts . || true ;
+                  cp -ra {{ ansible_user_dir }}/ci-framework-data/artifacts . ;
                   cp -ra {{ ansible_user_dir }}/ci-framework-data/tests . || true ;
 
             - name: Get SELinux listing
diff --git a/plugins/modules/verify_pulled_report_crio.py b/plugins/modules/verify_pulled_report_crio.py
@@ -20,7 +20,7 @@
     C(summary.mirror_rules).
   - When images carry a C(node) field, evidence is matched against the
     specific node's CRI-O log first.  If the digest is only found on a
-    different node the status is set to C(MISMATCH_CROSS_NODE).
+    different node the entry is counted as a cross-node mismatch.
   - Log files are expected to follow the C(<node-name>.crio.log) naming
     convention produced by the role task.
 
@@ -88,7 +88,7 @@
 cross_node_entries:
   description: >-
     Image rows where evidence was found only on a different node
-    than where the pod ran (C(MISMATCH_CROSS_NODE)).
+    than where the pod ran.
   type: int
   returned: always
 nodes_with_evidence:
@@ -240,29 +240,16 @@ def run_module():
 
         if node_local_hit:
             actual_uri = per_node_evidence[img_node][img_sha]
-            actual_domain = _apply_evidence(img, actual_uri, img_node, trusted_mirrors)
-            expected_domain = img.get("expected_pull_location") or ""
-            img["verification_status"] = (
-                "MATCH" if actual_domain == expected_domain else "MISMATCH"
-            )
+            _apply_evidence(img, actual_uri, img_node, trusted_mirrors)
         elif img_sha in global_evidence:
             actual_uri, evidence_node = global_evidence[img_sha]
-            actual_domain = _apply_evidence(
-                img, actual_uri, evidence_node, trusted_mirrors
-            )
-            expected_domain = img.get("expected_pull_location") or ""
+            _apply_evidence(img, actual_uri, evidence_node, trusted_mirrors)
             if img_node:
-                img["verification_status"] = "MISMATCH_CROSS_NODE"
                 cross_node_entries += 1
-            else:
-                img["verification_status"] = (
-                    "MATCH" if actual_domain == expected_domain else "MISMATCH"
-                )
         else:
             img["node_verified_image_origin"] = "cached/unknown"
             img["log_evidence_uri"] = None
             img["log_evidence_node"] = None
-            img["verification_status"] = "NOT_FOUND_IN_LOGS"
 
     nodes_with_evidence = sorted(n for n, ev in per_node_evidence.items() if ev)
     result = dict(
diff --git a/roles/env_op_images/README.md b/roles/env_op_images/README.md
@@ -5,7 +5,7 @@ A role to gather the container images used in the openstack deployment with spec
 * `cifmw_env_op_images_dir`: (String) Directory where the operator_images.yaml will be stored. Defaults to `~/ci-framework-data/artifacts`
 * `cifmw_env_op_images_file`: (String) Name of the file storing the operator images and tags. Defaults to `operator_images.yaml`
 * `cifmw_env_op_images_pulled_report_file` / `cifmw_env_op_images_pulled_report_path`: Pulled-images policy report (ICSP/IDMS + pod image refs).
-* `cifmw_env_op_images_verified_report_file` / `cifmw_env_op_images_verified_report_path`: Output path for the CRI-O-enriched report. After the pulled report runs, fetches `oc adm node-logs NODE -u crio` per node, then writes this file with digest-level CRI-O fields (`node_verified_image_origin`, `log_evidence_uri`, `verification_status`).
+* `cifmw_env_op_images_verified_report_file` / `cifmw_env_op_images_verified_report_path`: Output path for the CRI-O-enriched report. After the pulled report runs, fetches `oc adm node-logs NODE -u crio` per node, then writes this file with digest-level CRI-O fields (`node_verified_image_origin`, `log_evidence_uri`, `log_evidence_node`).
 * `cifmw_env_op_images_crio_logs_dir`: Directory for per-node `*.crio.log` files used during verification.
 
 ## Examples
diff --git a/roles/env_op_images/tasks/pulled_images_report.yml b/roles/env_op_images/tasks/pulled_images_report.yml
@@ -37,22 +37,27 @@
         state: directory
         mode: "0755"
 
-    # Legacy OpenShift mirror CRD; empty or error is OK (parsed as no items).
     - name: Get ICSP mirror rules
       ansible.builtin.command:
         cmd: oc get imagecontentsourcepolicy -o json
       register: _pulled_report_icsp
       failed_when: false
 
-    # Preferred / current mirror CRD alongside ICSP.
     - name: Get IDMS mirror rules
       ansible.builtin.command:
         cmd: oc get imagedigestmirrorset -o json
       register: _pulled_report_idms
       failed_when: false
 
-    # One flat list of rules for templating: each mirror list entry becomes
-    # its own row so prefix matching can use the same loop for all mirrors.
+    # Flatten ICSP repositoryDigestMirrors and IDMS imageDigestMirrors into
+    # a single list of {source, mirror} pairs.  A source with N mirrors
+    # produces N entries so the Jinja template can prefix-match every
+    # mirror in one loop.
+    # Example output (_pulled_report_mirror_mappings):
+    #   - source: registry.redhat.io/openstack-k8s-operators
+    #     mirror: quay.example.com/rh-osbs/openstack-k8s-operators
+    #   - source: registry.redhat.io/openstack-k8s-operators
+    #     mirror: internal-registry.corp.net/openstack-k8s-operators
     - name: Build source-to-mirror mapping from ICSP/IDMS
       vars:
         _icsp_items: >-
@@ -85,15 +90,6 @@
       ansible.builtin.set_fact:
         _pulled_report_mirror_mappings: "{{ _mappings | trim | from_yaml }}"
 
-    - name: Report ICSP/IDMS mirror rules found
-      when: _pulled_report_mirror_mappings | length > 0
-      ansible.builtin.debug:
-        msg: >-
-          {{ item.source }} -> {{ item.mirror }}
-      loop: "{{ _pulled_report_mirror_mappings }}"
-      loop_control:
-        label: "{{ item.source }}"
-
     - name: Warn if no ICSP/IDMS mirror rules found
       when: _pulled_report_mirror_mappings | length == 0
       ansible.builtin.debug:
@@ -115,58 +111,10 @@
         label: "{{ item }}"
       failed_when: false
 
-    # Flatten all loop results, then one report row per container + initContainer.
     - name: Build per-pod pulled images report
-      vars:
-        _report: >-
-          {% set entries = [] %}
-          {% set all_pods = _pulled_report_pods.results |
-                            map(attribute='resources', default=[]) | flatten %}
-          {% for pod in all_pods %}
-          {%   set pod_labels = pod.metadata.labels | default({}) %}
-          {%   set operator_name = pod_labels.get('openstack.org/operator-name', '') %}
-          {%   set app_label = pod_labels.get('app', pod_labels.get('app.kubernetes.io/name', '')) %}
-          {# Build a single list so regular and init containers share one loop #}
-          {%   set image_statuses = [] %}
-          {%   for cs in pod.status.containerStatuses | default([]) %}
-          {%     set _ = image_statuses.append({'cs': cs, 'type': 'container'}) %}
-          {%   endfor %}
-          {%   for cs in pod.status.initContainerStatuses | default([]) %}
-          {%     set _ = image_statuses.append({'cs': cs, 'type': 'init_container'}) %}
-          {%   endfor %}
-          {%   for entry in image_statuses %}
-          {%     set cs = entry.cs %}
-          {%     set image = cs.image | default('unknown') %}
-          {%     set image_id = cs.imageID | default('') %}
-          {%     set image_repo = image.split('@')[0].split(':')[0] %}
-          {%     set image_name = image_repo.split('/')[-1] %}
-          {# Default: host = first path segment of image ref; basis = source #}
-          {%     set match = namespace(host=image.split('/')[0], expected_pull_basis='source') %}
-          {# First mapping where image startswith source: use mirror registry host #}
-          {%     for m in _pulled_report_mirror_mappings if m.source is defined and m.source and image.startswith(m.source) %}
-          {%       if loop.first %}
-          {%         set match.host = m.mirror.split('/')[0] %}
-          {%         set match.expected_pull_basis = 'mirror' %}
-          {%       endif %}
-          {%     endfor %}
-          {%     set _ = entries.append({
-                   'namespace': pod.metadata.namespace,
-                   'pod': pod.metadata.name,
-                   'node': pod.spec.get('nodeName', 'unknown'),
-                   'container': cs.name,
-                   'container_type': entry.type,
-                   'operator': operator_name if operator_name else 'N/A',
-                   'app': app_label if app_label else image_name,
-                   'image': image,
-                   'image_id': image_id,
-                   'expected_pull_location': match.host,
-                   'expected_pull_basis': match.expected_pull_basis
-                 }) %}
-          {%   endfor %}
-          {% endfor %}
-          {{ entries }}
       ansible.builtin.set_fact:
-        _pulled_images_report: "{{ _report | trim | from_yaml }}"
+        _pulled_images_report: >-
+          {{ lookup('template', 'pulled_images_report.j2') | trim | from_yaml }}
 
     # Counts for the YAML artifact summary section.
     - name: Build report summary
@@ -197,31 +145,11 @@
         content: "{{ _full_report | to_nice_yaml }}"
         mode: "0644"
 
-    # Console visibility: split rows by how ICSP/IDMS classification turned out.
-    - name: Images with expected_pull_basis source
+    - name: Pulled images report summary
       ansible.builtin.debug:
         msg: >-
-          [{{ item.app }}] {{ item.image }} ->
-          expected pull location {{ item.expected_pull_location }}
-          ({{ item.container_type }}: {{ item.container }}
-          | operator: {{ item.operator }}
-          | pod: {{ item.namespace }}/{{ item.pod }})
-      loop: >-
-        {{ _pulled_images_report |
-           selectattr('expected_pull_basis', 'equalto', 'source') | list }}
-      loop_control:
-        label: "{{ item.app }}/{{ item.container }}"
-
-    - name: Images with expected_pull_basis mirror
-      ansible.builtin.debug:
-        msg: >-
-          [{{ item.app }}] {{ item.image }} ->
-          expected pull location {{ item.expected_pull_location }}
-          ({{ item.container_type }}: {{ item.container }}
-          | operator: {{ item.operator }}
-          | pod: {{ item.namespace }}/{{ item.pod }})
-      loop: >-
-        {{ _pulled_images_report |
-           selectattr('expected_pull_basis', 'equalto', 'mirror') | list }}
-      loop_control:
-        label: "{{ item.app }}/{{ item.container }}"
+          Pulled images report: {{ _pulled_report_summary.total_containers }} containers
+          ({{ _pulled_report_summary.containers_expected_basis_mirror }} mirror,
+          {{ _pulled_report_summary.containers_expected_basis_source }} source),
+          {{ _pulled_report_summary.mirror_rules_found }} mirror rules.
+          Full report: {{ cifmw_env_op_images_pulled_report_path }}
diff --git a/roles/env_op_images/tasks/verify_pulled_report_crio.yml b/roles/env_op_images/tasks/verify_pulled_report_crio.yml
@@ -64,15 +64,26 @@
         msg: >-
           oc get nodes failed (rc={{ _verify_crio_nodes_json.rc }}); cannot fetch CRI-O logs.
 
-    # Filename is sanitised to avoid path-traversal with unusual node names.
     - name: Fetch CRI-O unit logs per node
-      ansible.builtin.shell: >-
-        oc adm node-logs "{{ item }}" -u crio --since=-2h >
-        "{{ cifmw_env_op_images_crio_logs_dir }}/{{ item | regex_replace('[^A-Za-z0-9._-]+', '_') }}.crio.log"
+      ansible.builtin.command:
+        cmd: >-
+          oc adm node-logs "{{ item }}" -u crio
+          --since=-4h
       loop: "{{ _verify_crio_node_names | default([]) }}"
       timeout: 120
       register: _verify_crio_fetch
 
+    # Filename is sanitised to avoid path-traversal with unusual node names.
+    - name: Write CRI-O logs to files per node
+      ansible.builtin.copy:
+        dest: "{{ cifmw_env_op_images_crio_logs_dir }}/{{ item.item | regex_replace('[^A-Za-z0-9._-]+', '_') }}.crio.log"
+        content: "{{ item.stdout }}"
+        mode: "0644"
+      loop: "{{ _verify_crio_fetch.results | default([]) }}"
+      loop_control:
+        label: "{{ item.item | default('') }}"
+      when: item.rc | default(1) == 0
+
     # Non-fatal: some nodes may be unreachable (e.g. NotReady).
     - name: Warn when node log fetch failed for a node
       when: item.rc | default(0) != 0
diff --git a/roles/env_op_images/templates/pulled_images_report.j2 b/roles/env_op_images/templates/pulled_images_report.j2
@@ -0,0 +1,40 @@
+{# Produces a YAML list of per-container image entries.
+   Inputs (Ansible facts):
+     _pulled_report_pods            – register from k8s_info pod queries
+     _pulled_report_mirror_mappings – flat list of {source, mirror} dicts
+#}
+{% set entries = [] %}
+{% set all_pods = _pulled_report_pods.results |
+                  map(attribute='resources', default=[]) | flatten %}
+{% for pod in all_pods %}
+{%   set image_statuses = [] %}
+{%   for cs in pod.status.containerStatuses | default([]) %}
+{%     set _ = image_statuses.append({'cs': cs, 'type': 'container'}) %}
+{%   endfor %}
+{%   for cs in pod.status.initContainerStatuses | default([]) %}
+{%     set _ = image_statuses.append({'cs': cs, 'type': 'init_container'}) %}
+{%   endfor %}
+{%   for entry in image_statuses %}
+{%     set cs = entry.cs %}
+{%     set image = cs.image | default('unknown') %}
+{%     set image_id = cs.imageID | default('') %}
+{%     set match = namespace(host=image.split('/')[0], expected_pull_basis='source') %}
+{%     for m in _pulled_report_mirror_mappings if m.source is defined and m.source and image.startswith(m.source) %}
+{%       if loop.first %}
+{%         set match.host = m.mirror.split('/')[0] %}
+{%         set match.expected_pull_basis = 'mirror' %}
+{%       endif %}
+{%     endfor %}
+{%     set _ = entries.append({
+         'namespace': pod.metadata.namespace,
+         'pod': pod.metadata.name,
+         'node': pod.spec.get('nodeName', 'unknown'),
+         'container': cs.name,
+         'image': image,
+         'image_id': image_id,
+         'expected_pull_location': match.host,
+         'expected_pull_basis': match.expected_pull_basis
+       }) %}
+{%   endfor %}
+{% endfor %}
+{{ entries }}
