feat: support multiple-network deployments

README.md

@@ -249,6 +249,11 @@ Every Ansible deployment automatically deploys an observability stack alongside
       - `tmp/local-run-DD-MM-YYYY-HH-MM.log` for local deployments
       - `tmp/ansible-run-DD-MM-YYYY-HH-MM.log` for Ansible deployments
     - Example: `NETWORK_DIR=local-devnet ./spin-node.sh --node all --logs`
+19. `--network` sets the network name label attached to every metric and log stream scraped by the observability stack (Ansible mode only).
+   - Default: `devnet-3`, set in `parse-env.sh` after argument parsing
+   - Propagated to Ansible as the `network_name` variable, which is used in `prometheus.yml.j2` and `promtail.yml.j2` templates
+   - Appears as the `network` label on all Prometheus scrape targets (app, node_exporter, cadvisor) and all Promtail log streams, so you can filter by network in Grafana across multiple environments
+   - Example: `--network devnet-x`
 
 ### Preparing remote servers
 

ansible/playbooks/clean-node-data.yml

@@ -6,7 +6,7 @@
   connection: local
   gather_facts: no
   vars:
-    validator_config_file: "{{ genesis_dir }}/validator-config.yaml"
+    validator_config_file: "{{ local_validator_config_path | default(genesis_dir + '/validator-config.yaml') }}"
   tags:
     - zeam
     - ream

ansible/playbooks/deploy-nodes.yml

@@ -18,7 +18,7 @@
     - deploy
     - observability
   vars:
-    validator_config_file: "{{ genesis_dir }}/validator-config.yaml"
+    validator_config_file: "{{ local_validator_config_path | default(genesis_dir + '/validator-config.yaml') }}"
 
   tasks:
     - name: Validate validator-config.yaml exists
@@ -122,10 +122,7 @@
 
     - name: Sync validator-config.yaml to remote host
       copy:
-        # Use the expanded subnet config when --subnets was specified; fall back
-        # to the standard validator-config.yaml otherwise. The destination is
-        # always validator-config.yaml so client roles don't need to change.
-        src: "{{ local_genesis_dir }}/{{ validator_config_basename | default('validator-config.yaml') }}"
+        src: "{{ local_validator_config_path | default(local_genesis_dir + '/validator-config.yaml') }}"
         dest: "{{ genesis_dir }}/validator-config.yaml"
         mode: '0644'
         force: yes

ansible/playbooks/generate-genesis.yml

@@ -34,9 +34,19 @@
       set_fact:
         project_root: "{{ project_root_result.stdout }}"
 
+    - name: Copy custom validator config to genesis dir if different
+      copy:
+        src: "{{ local_validator_config_path }}"
+        dest: "{{ genesis_dir }}/validator-config.yaml"
+        mode: '0644'
+      when:
+        - local_validator_config_path is defined
+        - local_validator_config_path != genesis_dir + '/validator-config.yaml'
+
     - name: Run generate-genesis.sh script
       shell: |
-        cd "{{ project_root }}" && ./generate-genesis.sh "{{ genesis_dir }}" --mode ansible --offset {{ genesis_time_offset }}
+        cd "{{ project_root }}" && ./generate-genesis.sh "{{ genesis_dir }}" --mode ansible --offset {{ genesis_time_offset }}{% if local_validator_config_path is defined and local_validator_config_path != '' %} --validator-config "{{ local_validator_config_path }}"{% endif %}
+
       register: genesis_result
       args:
         executable: /bin/bash
@@ -55,7 +65,7 @@
 
     - name: Extract node names from validator-config.yaml
       shell: |
-        yq eval '.validators[].name' "{{ genesis_dir }}/validator-config.yaml"
+        yq eval '.validators[].name' "{{ local_validator_config_path | default(genesis_dir + '/validator-config.yaml') }}"
       register: node_names_raw
       changed_when: false
 

ansible/playbooks/helpers/deploy-single-node.yml

@@ -9,7 +9,7 @@
 - name: Set validator config file paths
   set_fact:
     actual_validator_config_file: "{{ genesis_dir }}/validator-config.yaml"
-    local_validator_config_file: "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    local_validator_config_file: "{{ local_validator_config_path }}"
 
 - name: Extract node configuration (from local config)
   shell: |

ansible/playbooks/prepare.yml

@@ -224,7 +224,7 @@
 
     - name: Read all node entries for this host from the active validator config
       vars:
-        _vc_file: "{{ _genesis_dir + '/' + (validator_config_basename | default('validator-config.yaml')) }}"
+        _vc_file: "{{ local_validator_config_path | default(_genesis_dir + '/validator-config.yaml') }}"
         _vc: "{{ lookup('file', _vc_file) | from_yaml }}"
         _entries: "{{ _vc.validators | selectattr('enrFields.ip', 'equalto', ansible_host) | list }}"
       set_fact:

ansible/playbooks/stop-nodes.yml

@@ -7,7 +7,7 @@
   connection: local
   gather_facts: yes
   vars:
-    validator_config_file: "{{ genesis_dir }}/validator-config.yaml"
+    validator_config_file: "{{ local_validator_config_path | default(genesis_dir + '/validator-config.yaml') }}"
 
   tasks:
     - name: Validate validator-config.yaml exists

ansible/roles/ethlambda/tasks/main.yml

@@ -31,7 +31,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: ethlambda_node_config
   changed_when: false
   delegate_to: localhost

ansible/roles/gean/tasks/main.yml

@@ -27,7 +27,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: gean_node_config
   changed_when: false
   delegate_to: localhost

ansible/roles/grandine/tasks/main.yml

@@ -31,7 +31,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: grandine_node_config
   changed_when: false
   delegate_to: localhost

ansible/roles/lantern/tasks/main.yml

@@ -22,7 +22,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: lantern_node_config
   changed_when: false
   delegate_to: localhost

ansible/roles/lighthouse/tasks/main.yml

@@ -30,7 +30,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: lighthouse_node_config
   changed_when: false
   delegate_to: localhost

ansible/roles/nlean/tasks/main.yml

@@ -27,7 +27,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: nlean_node_config
   changed_when: false
   delegate_to: localhost

ansible/roles/observability/tasks/main.yml

@@ -2,16 +2,18 @@
 # Observability role: Deploy cadvisor, node_exporter, prometheus, and promtail
 # alongside each lean node on remote hosts.
 
-- name: Extract metricsPort from validator-config.yaml
+- name: Find all nodes on this host
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .metricsPort" "{{ local_genesis_dir }}/validator-config.yaml"
-  register: obs_metrics_port_raw
+    yq eval '[.validators[] | select(.enrFields.ip == "{{ ansible_host }}") | {"name": .name, "metricsPort": .metricsPort}]' \
+      "{{ local_validator_config_path }}" -o=json
+  register: colocated_nodes_raw
   changed_when: false
   delegate_to: localhost
 
-- name: Set metricsPort fact
+- name: Set colocated nodes fact
   set_fact:
-    obs_metrics_port: "{{ obs_metrics_port_raw.stdout | trim }}"
+    colocated_nodes: "{{ colocated_nodes_raw.stdout | from_json }}"
+    is_primary_node: "{{ (colocated_nodes_raw.stdout | from_json)[0].name == node_name }}"
 
 - name: Create observability config directory
   file:
@@ -32,10 +34,13 @@
     mode: '0644'
 
 # --- cadvisor (always recreate to ensure correct flags) ---
+# Only the first colocated node on each machine manages infra containers
+# to avoid parallel rm+run races when multiple nodes share a host.
 
 - name: Remove existing cadvisor container
   command: docker rm -f cadvisor
   failed_when: false
+  when: is_primary_node
 
 - name: Start cadvisor container
   command: >-
@@ -55,12 +60,14 @@
     -v /var/lib/docker/:/var/lib/docker:ro
     {{ cadvisor_image }}
     --port={{ cadvisor_port }}
+  when: is_primary_node
 
 # --- node_exporter (always recreate to ensure correct flags) ---
 
 - name: Remove existing node_exporter container
   command: docker rm -f node_exporter
   failed_when: false
+  when: is_primary_node
 
 - name: Start node_exporter container
   command: >-
@@ -77,6 +84,7 @@
     --path.sysfs=/host/sys
     --path.rootfs=/rootfs
     --web.listen-address=0.0.0.0:{{ node_exporter_port }}
+  when: is_primary_node
 
 # --- prometheus (always recreate to pick up config/mount changes, data persists on host) ---
 
@@ -90,6 +98,7 @@
 - name: Remove existing prometheus container
   command: docker rm -f prometheus
   failed_when: false
+  when: is_primary_node
 
 - name: Start prometheus container
   command: >-
@@ -103,12 +112,14 @@
     --config.file=/etc/prometheus/prometheus.yml
     --storage.tsdb.retention.time=15d
     --web.listen-address=0.0.0.0:{{ prometheus_port }}
+  when: is_primary_node
 
 # --- promtail (always recreate to pick up config/mount changes) ---
 
 - name: Remove existing promtail container
   command: docker rm -f promtail
   failed_when: false
+  when: is_primary_node
 
 - name: Start promtail container
   command: >-
@@ -122,3 +133,4 @@
     {{ promtail_image }}
     -config.file=/etc/promtail/config.yml
     -server.http-listen-port={{ promtail_port }}
+  when: is_primary_node

ansible/roles/observability/templates/prometheus.yml.j2

@@ -2,22 +2,34 @@ global:
   scrape_interval: 15s
 
 scrape_configs:
-  - job_name: '{{ node_name }}'
+{% for node in colocated_nodes %}
+  - job_name: '{{ node.name }}'
     static_configs:
-      - targets: ['172.17.0.1:{{ obs_metrics_port }}']
+      - targets: ['172.17.0.1:{{ node.metricsPort }}']
         labels:
           type: 'app'
-          node_id: '{{ node_name }}'
+          instance: '{{ ansible_host }}'
+          network: '{{ network_name }}'
+          client_type: '{{ node.name.split("_")[0] }}'
+{% endfor %}
+  - job_name: 'node_exporter'
+    static_configs:
       - targets: ['172.17.0.1:{{ node_exporter_port }}']
         labels:
           type: 'node'
-          node_id: '{{ node_name }}'
+          instance: '{{ ansible_host }}'
+          network: '{{ network_name }}'
+  - job_name: 'cadvisor'
+    static_configs:
       - targets: ['172.17.0.1:{{ cadvisor_port }}']
         labels:
           type: 'docker'
-          node_id: '{{ node_name }}'
-    relabel_configs:
-      - source_labels: [node_id]
-        target_label: instance
+          instance: '{{ ansible_host }}'
+          network: '{{ network_name }}'
+    metric_relabel_configs:
+      - source_labels: [name]
+        regex: '([a-z]+)_.*'
+        target_label: client_type
+        replacement: '$1'
 remote_write:
   - url: {{ remote_write_url }}

ansible/roles/observability/templates/promtail.yml.j2

@@ -9,20 +9,25 @@ clients:
   - url: {{ loki_push_url }}
 
 scrape_configs:
-  - job_name: {{ node_name }}
+  - job_name: {{ ansible_host }}
     docker_sd_configs:
       - host: unix:///var/run/docker.sock
         refresh_interval: 5s
         filters:
           - name: name
-            values: ["{{ node_name }}"]
+            values:
+{% for node in colocated_nodes %}
+              - "{{ node.name }}"
+{% endfor %}
     relabel_configs:
       - source_labels: ['__meta_docker_container_name']
         regex: '/(.*)'
         target_label: 'container'
       - source_labels: ['__meta_docker_container_log_stream']
         target_label: 'stream'
-      - target_label: 'node'
-        replacement: '{{ node_name }}'
-      - target_label: 'host'
+      - source_labels: ['container']
+        target_label: 'node'
+      - target_label: 'instance'
         replacement: '{{ ansible_host }}'
+      - target_label: 'network'
+        replacement: '{{ network_name }}'

ansible/roles/peam/tasks/main.yml

@@ -27,7 +27,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: peam_node_config
   changed_when: false
   delegate_to: localhost
@@ -50,7 +50,7 @@
 
 - name: Extract local validator index from validator-config ordering
   shell: |
-    yq eval '.validators[].name' "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" | nl -v0 | awk '$2=="{{ node_name }}" {print $1; exit}'
+    yq eval '.validators[].name' "{{ local_validator_config_path }}" | nl -v0 | awk '$2=="{{ node_name }}" {print $1; exit}'
   register: peam_validator_index
   changed_when: false
   delegate_to: localhost
@@ -68,14 +68,14 @@
 
 - name: Extract total validator count from validator-config.yaml
   shell: |
-    yq eval '.validators[].count // 1' "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" | awk '{sum += $1} END {print sum + 0}'
+    yq eval '.validators[].count // 1' "{{ local_validator_config_path }}" | awk '{sum += $1} END {print sum + 0}'
   register: peam_total_validator_count
   changed_when: false
   delegate_to: localhost
 
 - name: Extract attestation committee count from validator-config.yaml
   shell: |
-    yq eval '.config.attestation_committee_count // 1' "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval '.config.attestation_committee_count // 1' "{{ local_validator_config_path }}"
   register: peam_attestation_committee_count_raw
   changed_when: false
   delegate_to: localhost

ansible/roles/qlean/tasks/main.yml

@@ -29,7 +29,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: qlean_node_config
   changed_when: false
   delegate_to: localhost

ansible/roles/ream/tasks/main.yml

@@ -30,7 +30,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: ream_node_config
   changed_when: false
   delegate_to: localhost

ansible/roles/zeam/tasks/main.yml

@@ -38,7 +38,7 @@
 
 - name: Extract node configuration from validator-config.yaml
   shell: |
-    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml"
+    yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ local_validator_config_path }}"
   register: node_config
   changed_when: false
   delegate_to: localhost