Blue/green deploys for OONI API services on Hetzner

ansible/deploy-ooniapi-gateway.yml

@@ -0,0 +1,7 @@
+---
+- name: Bootstrap the tier0 blue/green gateway on dedicated backend hosts
+  hosts: backend-fsn.ooni.org:backend-hel.ooni.org
+  become: true
+  roles:
+    - role: ooniapi_gateway
+      tags: ooniapi_gateway

ansible/roles/ooniapi_gateway/defaults/main.yml

@@ -0,0 +1,63 @@
+---
+# Domain the gateway vhost answers on for path-based routing. Mirrors the
+# ALB's `ooniapi_frontend_main_domain_name` in
+# tf/modules/ooniapi_frontend/main.tf (e.g. "api.prod.ooni.io").
+# No default: set explicitly per host in host_vars.
+ooniapi_gateway_primary_domain: ""
+
+# Any additional hostnames that should hit the same path-routed vhost.
+# Mirrors extra entries in `ooniapi_frontend_alternative_domains` that
+# aren't handled elsewhere (e.g. "api.ooni.org" in prod).
+ooniapi_gateway_extra_server_names: []
+
+# Suffix used for the ALB's secondary, host-header-only routing rules, e.g.
+# "oonirun.prod.ooni.io". Mirrors `direct_domain_suffix` in
+# tf/modules/ooniapi_frontend/main.tf ("${stage}.ooni.io").
+# No default: set explicitly per host in host_vars.
+ooniapi_gateway_direct_domain_suffix: ""
+
+# Name of the dehydrated cert (the directory under
+# ooniapi_gateway_cert_path) that covers ooniapi_gateway_primary_domain,
+# every "<service>.{{ ooniapi_gateway_direct_domain_suffix }}" name, and
+# ooniapi_gateway_extra_server_names as SANs.
+#
+# This role does NOT manage that cert -- issuing it means extending (or
+# adding to) the `dehydrated` role's ssl_domains on this host, which in turn
+# requires DNS for every one of those names to already resolve here. That's
+# a DNS-cutover-time step, not a day-1 bootstrap step, so it's deliberately
+# left manual. See the role README notes.
+ooniapi_gateway_cert_name: "{{ ooniapi_gateway_primary_domain }}"
+ooniapi_gateway_cert_path: /var/lib/dehydrated/certs/
+
+# Flip to true only once ooniapi_gateway_cert_name actually exists on disk.
+# Until then the vhost file is rendered but not installed, so this role is
+# safe to run ahead of the real DNS cutover.
+ooniapi_gateway_cert_ready: false
+
+# Services fronted by this gateway, and the two host ports their blue/green
+# slots are bound to. host_port_a/host_port_b must match the values passed
+# to that service's ooniapi_service_deployer module invocation in Terraform.
+#
+# Example:
+# ooniapi_gateway_services:
+#   - name: reverseproxy
+#     host_port_a: 18001
+#     host_port_b: 18002
+#   - name: oonirun
+#     host_port_a: 18011
+#     host_port_b: 18012
+ooniapi_gateway_services: []
+
+# Cross-cloud passthrough for tier0 services not yet migrated off AWS
+# (currently just testlists). Set to its public hostname to proxy those
+# paths there until testlists itself is migrated; leave empty to omit
+# those routes entirely.
+ooniapi_gateway_testlists_upstream: ""
+
+# Podman network shared by every service's containers on this host.
+ooniapi_gateway_network_name: ooniapi
+
+# SSH user the deploy CodeBuild job connects as. Must match
+# `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer
+# Terraform module.
+ooniapi_gateway_deploy_user: deploy

ansible/roles/ooniapi_gateway/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+- name: test ooniapi gateway nginx config
+  ansible.builtin.command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf
+  listen: reload ooniapi gateway nginx
+
+- name: reload ooniapi gateway nginx
+  ansible.builtin.service:
+    name: nginx
+    state: reloaded

ansible/roles/ooniapi_gateway/tasks/main.yml

@@ -0,0 +1,119 @@
+---
+# One-time (and idempotent-on-rerun) bootstrap for a host that will run
+# tier0 services via Podman Quadlet blue/green deploys, plus the nginx
+# gateway vhost that fronts them. See the Terraform side of this in
+# tf/modules/ooniapi_service_deployer (deploy_mode = "blue_green") and its
+# templates/buildspec_deploy.yml, which is what actually performs each
+# deploy -- this role only prepares the host for that job to run against.
+#
+# Runs as root (become: true at the play level). Podman state (networks,
+# secrets) is created in root's namespace to match the root-owned systemd
+# Quadlet units under /etc/containers/systemd/.
+
+- name: Ensure per-service state directories exist
+  ansible.builtin.file:
+    path: "/etc/ooniapi/{{ item.name }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  loop: "{{ ooniapi_gateway_services }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Check whether each service already has an active_slot marker
+  ansible.builtin.stat:
+    path: "/etc/ooniapi/{{ item.name }}/active_slot"
+  loop: "{{ ooniapi_gateway_services }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: ooniapi_gateway_active_slot_stat
+
+- name: Seed active_slot with "a" for services deploying for the first time
+  ansible.builtin.copy:
+    dest: "/etc/ooniapi/{{ item.item.name }}/active_slot"
+    content: "a"
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ ooniapi_gateway_active_slot_stat.results }}"
+  loop_control:
+    label: "{{ item.item.name }}"
+  when: not item.stat.exists
+
+- name: Check whether each service already has an nginx upstream conf
+  ansible.builtin.stat:
+    path: "/etc/nginx/conf.d/{{ item.name }}-upstream.conf"
+  loop: "{{ ooniapi_gateway_services }}"
+  loop_control:
+    label: "{{ item.name }}"
+  register: ooniapi_gateway_upstream_stat
+
+# Seeded with slot "a" up / "b" down, matching active_slot's "a" default
+# above, so the very first `nginx -t` (run when installing the gateway
+# vhost below) succeeds before that service's first real deploy has run.
+- name: Seed a placeholder upstream conf for services deploying for the first time
+  ansible.builtin.copy:
+    dest: "/etc/nginx/conf.d/{{ item.item.name }}-upstream.conf"
+    content: |
+      upstream {{ item.item.name }} {
+          server 127.0.0.1:{{ item.item.host_port_a }};
+          server 127.0.0.1:{{ item.item.host_port_b }} down;
+      }
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ ooniapi_gateway_upstream_stat.results }}"
+  loop_control:
+    label: "{{ item.item.name }}"
+  when: not item.stat.exists
+  notify: reload ooniapi gateway nginx
+
+- name: Check whether the podman network already exists
+  ansible.builtin.command: podman network exists {{ ooniapi_gateway_network_name }}
+  register: ooniapi_gateway_network_check
+  changed_when: false
+  failed_when: false
+
+- name: Create the shared podman network
+  ansible.builtin.command: podman network create {{ ooniapi_gateway_network_name }}
+  when: ooniapi_gateway_network_check.rc != 0
+
+- name: Ensure sudoers.d directory exists
+  ansible.builtin.file:
+    path: /etc/sudoers.d
+    state: directory
+    owner: root
+    group: root
+
+- name: Install scoped sudoers rule for the deploy user
+  ansible.builtin.template:
+    src: sudoers-ooniapi-deploy.j2
+    dest: /etc/sudoers.d/90-ooniapi-deploy
+    owner: root
+    group: root
+    mode: "0440"
+    validate: "visudo -cf %s"
+
+- name: Render the gateway vhost
+  ansible.builtin.template:
+    src: gateway.conf.j2
+    dest: /etc/nginx/conf.d/ooniapi-gateway.conf.pending
+    owner: root
+    group: root
+    mode: "0644"
+
+# Split into "render" + "install" so a bad template is visible
+# (.conf.pending) without ever being loaded by nginx, and so the vhost is
+# never installed at all until ooniapi_gateway_cert_ready is true -- see
+# defaults/main.yml for why that's a separate, deliberate step.
+- name: Install the gateway vhost now that its cert is ready
+  ansible.builtin.copy:
+    remote_src: true
+    src: /etc/nginx/conf.d/ooniapi-gateway.conf.pending
+    dest: /etc/nginx/conf.d/ooniapi-gateway.conf
+    owner: root
+    group: root
+    mode: "0644"
+  when: ooniapi_gateway_cert_ready
+  notify: reload ooniapi gateway nginx

ansible/roles/ooniapi_gateway/templates/gateway.conf.j2

@@ -0,0 +1,167 @@
+# Managed by ansible - roles/ooniapi_gateway/templates/gateway.conf.j2
+#
+# Single "gateway" vhost for the OONI tier0 API on this dedicated host,
+# mirroring (1:1, by design) the AWS ALB listener rules in
+# tf/modules/ooniapi_frontend/main.tf. Only rules for services actually
+# running blue/green here ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }})
+# are handled; everything unmatched falls through to "reverseproxy", exactly
+# like the ALB's default_action.
+#
+# Each service's own `<service>-upstream.conf` (the `upstream <service> {
+# ... }` block referenced below via proxy_pass) is dropped into this same
+# conf.d directory by that service's blue/green deploy job, not by this
+# role -- see ooniapi_service_deployer's buildspec_deploy.yml. A service's
+# location block here will fail nginx -t until that service has deployed at
+# least once.
+
+{% set service_names = ooniapi_gateway_services | map(attribute='name') | list %}
+
+# anonymize ipaddr (same scheme as the legacy ooni-api vhost)
+map $remote_addr $ooniapi_gateway_remote_addr_anon {
+  ~(?P<ip>\d+\.\d+\.\d+)\.    $ip.0;
+  ~(?P<ip>[^:]+:[^:]+):       $ip::;
+  default                     0.0.0.0;
+}
+
+log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon [$time_local] '
+    '"$request" $status $body_bytes_sent rt:$request_time "$http_referer" "$http_user_agent"';
+
+server {
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+  server_name {{ ([ooniapi_gateway_primary_domain] + ooniapi_gateway_extra_server_names) | join(' ') }};
+
+  access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt;
+  error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info;
+
+  client_max_body_size 200M; # for measurement POST, matches the legacy API vhost
+
+  ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem;
+  ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem;
+  ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem;
+
+  # https://ssl-config.mozilla.org/#server=nginx&config=intermediate -- kept
+  # inline (rather than shared include) so this role has no dependency on
+  # how nginx itself got installed on this host.
+  ssl_protocols TLSv1.2 TLSv1.3;
+  ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
+  ssl_prefer_server_ciphers off;
+  ssl_session_timeout 5m;
+  ssl_session_cache shared:MozSSL:30m;
+  ssl_session_tickets off;
+  ssl_stapling on;
+  ssl_stapling_verify on;
+
+  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
+  add_header X-Frame-Options DENY always;
+  add_header X-Content-Type-Options nosniff always;
+
+  proxy_http_version 1.1;
+  proxy_set_header Host $http_host;
+  proxy_set_header X-Real-IP $remote_addr;
+  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  proxy_set_header X-Forwarded-Proto $scheme;
+  proxy_read_timeout 900;
+
+{% if 'ooniauth' in service_names %}
+  ## --- ooniauth --- (ALB priority 108)
+  location ^~ /api/v2/ooniauth/ { proxy_pass http://ooniauth; }
+  location = /api/v1/user_register { proxy_pass http://ooniauth; }
+  location = /api/v1/user_login { proxy_pass http://ooniauth; }
+  location = /api/v1/user_refresh_token { proxy_pass http://ooniauth; }
+  location = /api/_/account_metadata { proxy_pass http://ooniauth; }
+{% endif %}
+
+{% if 'oonirun' in service_names %}
+  ## --- oonirun --- (ALB priority 110)
+  location ^~ /api/v2/oonirun/ { proxy_pass http://oonirun; }
+{% endif %}
+
+{% if 'ooniprobe' in service_names %}
+  ## --- ooniprobe --- (ALB priorities 120-123)
+  location ^~ /api/v2/ooniprobe/ { proxy_pass http://ooniprobe; }
+  location = /api/v1/login { proxy_pass http://ooniprobe; }
+  location = /api/v1/register { proxy_pass http://ooniprobe; }
+  location ^~ /api/v1/update/ { proxy_pass http://ooniprobe; }
+  location ^~ /api/v1/check-in { proxy_pass http://ooniprobe; }
+  location ^~ /api/v1/test-helpers { proxy_pass http://ooniprobe; }
+  location = /api/v1/test-list/urls { proxy_pass http://ooniprobe; }
+  location ^~ /report { proxy_pass http://ooniprobe; }
+  location = /api/_/show_countries_prioritization { proxy_pass http://ooniprobe; }
+  location = /api/_/debug_prioritization { proxy_pass http://ooniprobe; }
+  location ^~ /api/v1/manifest { proxy_pass http://ooniprobe; }
+  location ^~ /api/v1/sign_credential { proxy_pass http://ooniprobe; }
+  location ^~ /api/v1/submit_measurement { proxy_pass http://ooniprobe; }
+  location ^~ /bouncer/net-tests { proxy_pass http://ooniprobe; }
+  location ^~ /api/v1/geolookup { proxy_pass http://ooniprobe; }
+  location ^~ /api/v1/collectors { proxy_pass http://ooniprobe; }
+  location = /api/v1/test-list/tor-targets { proxy_pass http://ooniprobe; }
+  location = /api/v1/test-list/psiphon-config { proxy_pass http://ooniprobe; }
+{% endif %}
+
+{% if 'oonifindings' in service_names %}
+  ## --- oonifindings --- (ALB priority 130)
+  location ^~ /api/v1/incidents/ { proxy_pass http://oonifindings; }
+{% endif %}
+
+{% if 'oonimeasurements' in service_names %}
+  ## --- oonimeasurements --- (ALB priorities 140-143)
+  location ^~ /api/v1/measurements/ { proxy_pass http://oonimeasurements; }
+  location = /api/v1/raw_measurement { proxy_pass http://oonimeasurements; }
+  location = /api/v1/measurement_meta { proxy_pass http://oonimeasurements; }
+  location = /api/v1/measurements { proxy_pass http://oonimeasurements; }
+  location = /api/v1/torsf_stats { proxy_pass http://oonimeasurements; }
+  location = /api/v1/aggregation { proxy_pass http://oonimeasurements; }
+  location ^~ /api/v1/aggregation/ { proxy_pass http://oonimeasurements; }
+  location = /api/v1/observations { proxy_pass http://oonimeasurements; }
+  location = /api/v1/analysis { proxy_pass http://oonimeasurements; }
+  location = /api/v1/detector/changepoints { proxy_pass http://oonimeasurements; }
+{% endif %}
+
+{% if ooniapi_gateway_testlists_upstream %}
+  ## --- testlists (ALB priority 144; not yet migrated off AWS, proxied
+  ## cross-cloud until it is) ---
+  location ^~ /api/_/url-submission/test-list/ { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; }
+  location = /api/_/url-priorities/list { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; }
+  location = /api/_/url-priorities/update { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; }
+  location = /api/v1/url-submission/submit { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; }
+  location = /api/v1/url-submission/update-url { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; }
+{% endif %}
+
+  ## --- default: everything else goes to reverseproxy, same as the ALB's
+  ## default_action. Covers legacy/unmigrated paths (oonith is NOT among
+  ## them -- it's routed by host_header on *.th.ooni.org, a hostname this
+  ## vhost doesn't answer for, so it never reaches here either way). ---
+  location / {
+    proxy_pass http://reverseproxy;
+  }
+}
+
+{% for service in service_names %}
+{% if service != 'reverseproxy' %}
+server {
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+  server_name {{ service }}.{{ ooniapi_gateway_direct_domain_suffix }};
+
+  access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt;
+  error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info;
+
+  ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem;
+  ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem;
+  ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem;
+  ssl_protocols TLSv1.2 TLSv1.3;
+
+  proxy_http_version 1.1;
+  proxy_set_header Host $http_host;
+  proxy_set_header X-Real-IP $remote_addr;
+  proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+  proxy_set_header X-Forwarded-Proto $scheme;
+  proxy_read_timeout 900;
+
+  location / {
+    proxy_pass http://{{ service }};
+  }
+}
+{% endif %}
+{% endfor %}

ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2

@@ -0,0 +1,23 @@
+# Managed by ansible - roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2
+#
+# Scoped to exactly the remote commands issued by
+# tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml over
+# SSH as {{ ooniapi_gateway_deploy_user }}. Intentionally narrower than the
+# blanket "NOPASSWD:ALL" pattern used for the deploy user in other roles
+# (e.g. roles/anonc) -- this user is driven by an automated CI job over the
+# network, not a human operator.
+#
+# Caveat: sudoers glob matching (fnmatch without FNM_PATHNAME) lets "*"
+# match "/". The mv sources below are always files this same job just
+# scp'd into /tmp itself, so this is a theoretical rather than practical
+# widening of scope, but worth knowing if you're auditing this file.
+
+Cmnd_Alias OONIAPI_DEPLOY_MV_UNIT = /usr/bin/mv /tmp/*.container /etc/containers/systemd/*.container
+Cmnd_Alias OONIAPI_DEPLOY_MV_UPSTREAM = /usr/bin/mv /tmp/*-upstream.conf /etc/nginx/conf.d/*-upstream.conf
+Cmnd_Alias OONIAPI_DEPLOY_SYSTEMCTL = /usr/bin/systemctl daemon-reload, /usr/bin/systemctl restart *-a.service, /usr/bin/systemctl restart *-b.service, /usr/bin/systemctl reload nginx
+Cmnd_Alias OONIAPI_DEPLOY_NGINX_TEST = /usr/sbin/nginx -t
+Cmnd_Alias OONIAPI_DEPLOY_MARKER = /usr/bin/tee /etc/ooniapi/*/active_slot
+Cmnd_Alias OONIAPI_DEPLOY_PODMAN_SECRET = /usr/bin/podman secret create --replace * -
+
+{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_MV_UNIT, OONIAPI_DEPLOY_MV_UPSTREAM, OONIAPI_DEPLOY_SYSTEMCTL, OONIAPI_DEPLOY_NGINX_TEST, OONIAPI_DEPLOY_MARKER
+{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_PODMAN_SECRET