[reproducer] Replace all reproducer vars with zuul vars

Some variables are missing, especially related to the registry
credentials. That complicate the situation, where some keys
might not exists in reproducer file. Some of the variables
are jinja2 template, which are "translated" by Ansible.
Alternative way was to use yq command like:

    yq eval-all -P 'select(fileIndex == 0) * select(fileIndex == 1)' \
        /tmp/reproducer-variables.yml \
        /home/zuul/configs/zuul_vars.yaml > merged.yaml

The yq is available in latest version available for current python3
version installed on hypervisor. In that case, we can not use
the yq tool, but use a python script.

Co-authored-by: Claude (AI Assistant) claude@anthropic.com

Signed-off-by: Daniel Pawlik <dpawlik@redhat.com>

Author: danpawlik

roles/reproducer/files/merge_yaml_override.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+#
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Merge two YAML files whose root is a mapping: keys from OVERRIDE replace BASE
+# recursively for nested mappings; non-dict values (scalars, lists) are
+# replaced entirely by OVERRIDE. Requires PyYAML (python3-pyyaml / python3-yaml
+# on RPM systems).
+
+from __future__ import annotations
+
+import sys
+
+try:
+    import yaml
+except ImportError:
+    sys.stderr.write(
+        "merge_yaml_override.py: PyYAML is required "
+        "(e.g. dnf install python3-pyyaml or pip install pyyaml)\n"
+    )
+    sys.exit(1)
+
+
+class DoubleQuoteDumper(yaml.SafeDumper):
+    """YAML SafeDumper subclass;
+    emits strings with double quotes when needed for readability."""
+
+    pass
+
+
+def _str_representer(dumper, data):
+    """Take a SafeDumper and a str;
+    return a YAML scalar node (literal | or double-quoted when needed)."""
+    if "\n" in data:
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+    if any(c in data for c in "{}[]%*&!@#"):
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')
+    return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+
+
+DoubleQuoteDumper.add_representer(str, _str_representer)
+
+
+def deep_merge(base: object, override: object) -> object:
+    """Take two parsed values;
+    return merge where both are dicts (recursive), else override wins."""
+    if base is None:
+        return override
+    if override is None:
+        return base
+    if isinstance(base, dict) and isinstance(override, dict):
+        out = dict(base)
+        for key, val in override.items():
+            if key in out:
+                out[key] = deep_merge(out[key], val)
+            else:
+                out[key] = val
+        return out
+    return override
+
+
+def main() -> None:
+    """Read CLI paths (two or three args),
+    merge YAML mappings, write stdout or MERGED_OUT; exit on error."""
+    argc = len(sys.argv)
+    if argc not in (3, 4):
+        sys.stderr.write(
+            f"usage: {sys.argv[0]} FILE1_BASE.yml FILE2_OVERRIDE.yml [MERGED_OUT.yml]\n"
+            "  Deep-merge two YAML mappings: FILE2 wins on duplicate keys (nested dicts merged).\n"
+            "  Scalars and lists from FILE2 replace FILE1. If MERGED_OUT is omitted, print to stdout.\n"
+        )
+        sys.exit(2)
+
+    base_path, override_path = sys.argv[1], sys.argv[2]
+    out_path = sys.argv[3] if argc == 4 else None
+
+    with open(base_path, encoding="utf-8") as f:
+        base = yaml.safe_load(f)
+    with open(override_path, encoding="utf-8") as f:
+        override = yaml.safe_load(f)
+
+    if base is None:
+        base = {}
+    if override is None:
+        override = {}
+
+    if not isinstance(base, dict):
+        sys.stderr.write(
+            f"{base_path}: root must be a mapping, got {type(base).__name__}\n"
+        )
+        sys.exit(3)
+    if not isinstance(override, dict):
+        sys.stderr.write(
+            f"{override_path}: root must be a mapping, got {type(override).__name__}\n"
+        )
+        sys.exit(3)
+
+    merged = deep_merge(base, override)
+    dump_options = dict(
+        default_flow_style=False,
+        sort_keys=False,
+        allow_unicode=True,
+    )
+    if out_path is None:
+        yaml.dump(merged, sys.stdout, Dumper=DoubleQuoteDumper, **dump_options)
+    else:
+        with open(out_path, "w", encoding="utf-8") as out_f:
+            yaml.dump(merged, out_f, Dumper=DoubleQuoteDumper, **dump_options)
+
+
+if __name__ == "__main__":
+    main()

roles/reproducer/tasks/overwrite_zuul_vars.yml

@@ -15,46 +15,50 @@
 
 - name: Overwrite reproducer-variables.yml when ZIronic used
   when: _current_zuul_vars.stat.exists
+  vars:
+    temp_reproducer_var_path: /tmp/reproducer-variables.yml
+    temp_merged_reproducer_var_path: /tmp/merged-reproducer-variables.yml
   block:
-    - name: Dump reproducer-variables.yml to var
+    - name: Slurp reproducer-variables.yml to hypervisor
       ansible.builtin.slurp:
         src: "{{ cifmw_basedir }}/parameters/reproducer-variables.yml"
       register: reproducer_original
       delegate_to: controller-0
       no_log: "{{ cifmw_nolog | default(true) | bool }}"
 
-    - name: Dump zuul_vars.yaml to var
-      ansible.builtin.slurp:
-        src: "{{ ansible_user_dir }}/configs/zuul_vars.yaml"
-      register: zuul_job_vars
+    - name: Create temp file reproducer-variables.yml on hypervisor
+      ansible.builtin.copy:
+        content: "{{ reproducer_original.content | b64decode }}"
+        dest: "{{ temp_reproducer_var_path }}"
+        mode: "0664"
       no_log: "{{ cifmw_nolog | default(true) | bool }}"
 
-    - name: Load current job zuul_vars.yaml
-      ansible.builtin.include_vars:
-        file: "{{ ansible_user_dir }}/configs/zuul_vars.yaml"
-        name: _current_zuul_vars_include
+    - name: Copy merge yamls script
+      become: true
+      ansible.builtin.copy:
+        src: merge_yaml_override.py
+        dest: /usr/local/bin/merge_yaml_override
+        mode: "0755"
 
-    - name: Decode reproducer-variables.yml content
-      ansible.builtin.set_fact:
-        reproducer_vars_content: "{{ reproducer_original.content | b64decode | from_yaml }}"
+    - name: Execute merge script
+      ansible.builtin.shell: >
+        python3 /usr/local/bin/merge_yaml_override
+        {{ temp_reproducer_var_path }}
+        {{ ansible_user_dir }}/configs/zuul_vars.yaml >
+        {{ temp_merged_reproducer_var_path }}
       no_log: "{{ cifmw_nolog | default(true) | bool }}"
 
-    - name: Filter zuul vars to only keys present in reproducer-variables
-      ansible.builtin.set_fact:
-        _zuul_vars_filtered: >-
-          {{
-            _current_zuul_vars_include | dict2items
-            | selectattr('key', 'in', reproducer_vars_content.keys())
-            | items2dict
-          }}
+    - name: Slurp merged reproducer-variables.yml from hypervisor
+      ansible.builtin.slurp:
+        src: "{{ temp_merged_reproducer_var_path }}"
+      register: merged_reproducer_slurp
       no_log: "{{ cifmw_nolog | default(true) | bool }}"
 
     - name: Write back merged reproducer-variables.yml
       ansible.builtin.copy:
-        content: >-
-          {{ reproducer_vars_content | combine(_zuul_vars_filtered, recursive=true) | to_nice_yaml }}
+        content: "{{ merged_reproducer_slurp.content | b64decode }}"
         dest: "{{ cifmw_basedir }}/parameters/reproducer-variables.yml"
-        mode: '0664'
+        mode: "0664"
         backup: true
       no_log: "{{ cifmw_nolog | default(true) | bool }}"
       delegate_to: controller-0