From 73f50b3a68ec61abec844593b36b85f2618f317f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 10 Jul 2026 16:38:53 +0200 Subject: [PATCH 1/3] PoC of blue-green deployer to dedicated host --- tf/modules/ooniapi_service_deployer/main.tf | 258 +++++++++++++++++- .../templates/buildspec_deploy.yml | 92 +++++++ .../templates/nginx_upstream.conf.tftpl | 4 + .../templates/quadlet.container.tftpl | 23 ++ .../ooniapi_service_deployer/variables.tf | 105 ++++++- 5 files changed, 463 insertions(+), 19 deletions(-) create mode 100644 tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml create mode 100644 tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl create mode 100644 tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl diff --git a/tf/modules/ooniapi_service_deployer/main.tf b/tf/modules/ooniapi_service_deployer/main.tf index fb32fefd..b95722b7 100755 --- a/tf/modules/ooniapi_service_deployer/main.tf +++ b/tf/modules/ooniapi_service_deployer/main.tf @@ -161,6 +161,209 @@ resource "aws_codebuild_project" "ooniapi" { } } +## Podman Quadlet blue/green deploy (deploy_mode = "blue_green") + +resource "aws_s3_object" "quadlet_unit" { + for_each = var.deploy_mode == "blue_green" ? { a = var.host_port_a, b = var.host_port_b } : {} + + bucket = var.quadlet_units_bucket + key = "${var.service_name}/${var.service_name}-${each.key}.container" + content_type = "text/plain" + + content = templatefile("${path.module}/templates/quadlet.container.tftpl", { + service_name = var.service_name + slot = each.key + host_port = each.value + container_port = var.container_port + network_name = var.network_name + env_vars = var.env_vars + secrets = var.secrets + }) +} + +resource "aws_s3_object" "nginx_upstream" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + bucket = var.quadlet_units_bucket + key = "${var.service_name}/${var.service_name}-upstream.conf" + content_type = "text/plain" + + content = templatefile("${path.module}/templates/nginx_upstream.conf.tftpl", { + service_name = var.service_name + host_port_a = var.host_port_a + host_port_b = var.host_port_b + }) +} + +resource "aws_iam_policy" "deploy" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + description = "Policy used in trust relationship with the blue/green deploy CodeBuild project" + name = "codebuild-deploy-${var.service_name}-${var.aws_region}" + path = "/service-role/" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = [ + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/codebuild/ooniapi-${var.service_name}-deploy", + "arn:aws:logs:${var.aws_region}:${local.account_id}:log-group:/aws/codebuild/ooniapi-${var.service_name}-deploy:*" + ] + }, + { + # required for CodeBuild to read the CodePipeline BuildArtifact + # (imagedefinitions.json), mirrors the grant on the build role + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetBucketAcl", + "s3:GetBucketLocation" + ] + Resource = [ + "arn:aws:s3:::${var.codepipeline_bucket}", + "arn:aws:s3:::${var.codepipeline_bucket}/*" + ] + }, + { + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = ["arn:aws:s3:::${var.quadlet_units_bucket}/${var.service_name}/*"] + }, + { + Effect = "Allow" + Action = ["secretsmanager:GetSecretValue"] + Resource = [ + var.deploy_ssh_key_secret_arn, + var.service_secrets_arn + ] + } + ] + }) +} + +resource "aws_iam_role" "deploy" { + count = var.deploy_mode == "blue_green" ? 1 : 0 + + assume_role_policy = </dev/null || (apt-get update -y && apt-get install -y jq) + + pre_build: + commands: + - aws secretsmanager get-secret-value --secret-id "$DEPLOY_SSH_KEY_SECRET_ARN" --query SecretString --output text > /tmp/deploy_key + - chmod 600 /tmp/deploy_key + - aws secretsmanager get-secret-value --secret-id "$SERVICE_SECRETS_ARN" --query SecretString --output text > /tmp/service_secrets.json + - IMAGE_URI=$(jq -r '.[0].imageUri' imagedefinitions.json) + - export IMAGE_TAG="${IMAGE_URI##*:}" + - echo "Deploying $SERVICE_NAME image tag $IMAGE_TAG to $DEPLOY_HOST_PRIMARY then $DEPLOY_HOST_SECONDARY" + + build: + commands: + - | + set -euo pipefail + + deploy_host() { + HOST="$1" + echo "=== $SERVICE_NAME: deploying to $HOST ===" + + ACTIVE_SLOT=$(ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "cat /etc/ooniapi/${SERVICE_NAME}/active_slot 2>/dev/null || echo a") + if [ "$ACTIVE_SLOT" = "a" ]; then + TARGET_SLOT=b + TARGET_PORT="$HOST_PORT_B" + else + TARGET_SLOT=a + TARGET_PORT="$HOST_PORT_A" + fi + echo "$SERVICE_NAME on $HOST: active slot is $ACTIVE_SLOT, deploying to slot $TARGET_SLOT (port $TARGET_PORT)" + + # a. + b. render/fetch the target slot's quadlet unit and install it + UNIT_FILE="${SERVICE_NAME}-${TARGET_SLOT}.container" + aws s3 cp "s3://${QUADLET_BUCKET}/${SERVICE_NAME}/${UNIT_FILE}" "/tmp/${UNIT_FILE}" + sed -i "s|__IMAGE_TAG__|${IMAGE_TAG}|g" "/tmp/${UNIT_FILE}" + + scp $SSH_OPTS "/tmp/${UNIT_FILE}" "${DEPLOY_SSH_USER}@${HOST}:/tmp/${UNIT_FILE}" + ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo mv /tmp/${UNIT_FILE} /etc/containers/systemd/${UNIT_FILE}" + + # c. push service secrets as podman secrets (values travel over + # stdin, never interpolated into the remote command string) + for KEY in $(jq -r 'keys[]' /tmp/service_secrets.json); do + VALUE=$(jq -r --arg k "$KEY" '.[$k]' /tmp/service_secrets.json) + printf '%s' "$VALUE" | ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "podman secret create --replace ${KEY} -" + done + + # d. reload systemd and (re)start only the target slot + ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo systemctl daemon-reload && sudo systemctl restart ${SERVICE_NAME}-${TARGET_SLOT}.service" + + # e. health-check the target slot directly, bypassing nginx + HEALTHY="" + for i in $(seq 1 10); do + if ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "curl -sf -o /dev/null http://127.0.0.1:${TARGET_PORT}/health"; then + HEALTHY=1 + break + fi + sleep 2 + done + if [ -z "$HEALTHY" ]; then + echo "$SERVICE_NAME on $HOST: slot $TARGET_SLOT failed health check, aborting deploy" + exit 1 + fi + + # f. flip nginx to the target slot and record the new active slot + if [ "$TARGET_SLOT" = "a" ]; then + STATE_A="" + STATE_B="down" + else + STATE_A="down" + STATE_B="" + fi + UPSTREAM_FILE="${SERVICE_NAME}-upstream.conf" + aws s3 cp "s3://${QUADLET_BUCKET}/${SERVICE_NAME}/${UPSTREAM_FILE}" "/tmp/${UPSTREAM_FILE}" + sed -i "s|__STATE_A__|${STATE_A}|g; s|__STATE_B__|${STATE_B}|g" "/tmp/${UPSTREAM_FILE}" + + scp $SSH_OPTS "/tmp/${UPSTREAM_FILE}" "${DEPLOY_SSH_USER}@${HOST}:/tmp/${UPSTREAM_FILE}" + ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo mv /tmp/${UPSTREAM_FILE} /etc/nginx/conf.d/${UPSTREAM_FILE} && sudo nginx -t && sudo systemctl reload nginx" + ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "echo ${TARGET_SLOT} | sudo tee /etc/ooniapi/${SERVICE_NAME}/active_slot > /dev/null" + + echo "=== $SERVICE_NAME on $HOST: now serving from slot $TARGET_SLOT ===" + } + + deploy_host "$DEPLOY_HOST_PRIMARY" + deploy_host "$DEPLOY_HOST_SECONDARY" diff --git a/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl new file mode 100644 index 00000000..d239d393 --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/templates/nginx_upstream.conf.tftpl @@ -0,0 +1,4 @@ +upstream ${service_name} { + server 127.0.0.1:${host_port_a} __STATE_A__; + server 127.0.0.1:${host_port_b} __STATE_B__; +} diff --git a/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl b/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl new file mode 100644 index 00000000..f309d8c4 --- /dev/null +++ b/tf/modules/ooniapi_service_deployer/templates/quadlet.container.tftpl @@ -0,0 +1,23 @@ +[Unit] +Description=${service_name} (slot ${slot}) +After=network-online.target +Wants=network-online.target + +[Container] +ContainerName=${service_name}-${slot} +Image=docker.io/ooni/api-${service_name}:__IMAGE_TAG__ +Network=${network_name} +PublishPort=127.0.0.1:${host_port}:${container_port} +%{ for key, value in env_vars ~} +Environment="${key}=${value}" +%{ endfor ~} +%{ for secret_name in secrets ~} +Secret=${secret_name} +%{ endfor ~} + +[Service] +Restart=always +TimeoutStartSec=60 + +[Install] +WantedBy=multi-user.target diff --git a/tf/modules/ooniapi_service_deployer/variables.tf b/tf/modules/ooniapi_service_deployer/variables.tf index 73e0dc7e..67be2b53 100644 --- a/tf/modules/ooniapi_service_deployer/variables.tf +++ b/tf/modules/ooniapi_service_deployer/variables.tf @@ -30,15 +30,112 @@ variable "trigger_path" { description = "path filter for push changes which trigger the codepipeline eg. ooniapi/services/oonirun/**" } +variable "environment" { + description = "Deployment environment (e.g., prod, dev)" + type = string +} + +variable "deploy_mode" { + description = <<-EOF + Which Deploy stage implementation the pipeline uses: + - "ecs" (default) the existing ECS rolling-deploy stage. + - "blue_green" Podman Quadlet blue/green deploy to dedicated Hetzner + hosts, driven by a CodeBuild "Deploy" action over SSH. + This is opt-in per service so unmigrated services keep working unchanged. + EOF + type = string + default = "ecs" + + validation { + condition = contains(["ecs", "blue_green"], var.deploy_mode) + error_message = "deploy_mode must be either \"ecs\" or \"blue_green\"." + } +} + +# --- deploy_mode = "ecs" ----------------------------------------------- + variable "ecs_cluster_name" { - description = "id of the cluster to deploy into" + description = "id of the cluster to deploy into. Required when deploy_mode = \"ecs\"." + type = string + default = null } variable "ecs_service_name" { - description = "id of the service in the cluster to deploy" + description = "id of the service in the cluster to deploy. Required when deploy_mode = \"ecs\"." + type = string + default = null } -variable "environment" { - description = "Deployment environment (e.g., prod, dev)" +# --- deploy_mode = "blue_green" ----------------------------------------- + +variable "quadlet_units_bucket" { + description = "S3 bucket that rendered Quadlet unit files and the nginx upstream conf snippet are uploaded to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "host_port_a" { + description = "Host port bound to the \"a\" deploy slot. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "host_port_b" { + description = "Host port bound to the \"b\" deploy slot. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "container_port" { + description = "Port the service listens on inside the container. Required when deploy_mode = \"blue_green\"." + type = number + default = null +} + +variable "network_name" { + description = "Podman network the service's containers attach to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "secrets" { + description = "Names of Podman secrets (created/updated on the host during deploy) to mount into the container. Values live in var.service_secrets_arn." + type = list(string) + default = [] +} + +variable "env_vars" { + description = "Cleartext environment variables for the container. Same shape as ooniapi_service's task_environment (map(string))." + type = map(string) + default = {} +} + +variable "service_secrets_arn" { + description = "ARN of the Secrets Manager secret holding the service's runtime secrets as a flat JSON key/value object. Each key is pushed to the target hosts as a Podman secret during deploy. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_ssh_key_secret_arn" { + description = "ARN of the Secrets Manager secret holding the SSH private key the deploy CodeBuild job uses to reach the target hosts. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_host_primary" { + description = "Hostname/IP of the primary dedicated host to deploy to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_host_secondary" { + description = "Hostname/IP of the secondary dedicated host to deploy to. Required when deploy_mode = \"blue_green\"." + type = string + default = null +} + +variable "deploy_ssh_user" { + description = "SSH user the deploy job connects as on the target hosts." type = string + default = "deploy" } From d7bcb610b83a93f7b5364507ba502b8148235fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 10 Jul 2026 16:52:02 +0200 Subject: [PATCH 2/3] Add usage of the updated module from dev and prod --- tf/environments/dev/main.tf | 186 +++++++++++++++++++++++++++++++++++ tf/environments/prod/main.tf | 186 +++++++++++++++++++++++++++++++++++ 2 files changed, 372 insertions(+) diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 5d1f9e64..943e71a4 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -253,6 +253,96 @@ data "aws_ssm_parameter" "account_id_hashing_key" { name = "/oonidevops/secrets/ooni_services/account_id_hashing_key" } +### Blue/green deploy secrets (Podman Quadlet on dedicated Hetzner hosts) +# +# These mirror the values already passed as `task_secrets` to the ECS task +# definitions above, but consolidated into one JSON blob per service so the +# "blue_green" deploy_mode CodeBuild job can fetch them in a single +# secretsmanager:GetSecretValue call and push each key as a Podman secret. +# Values are pulled from the same underlying data sources used by the ECS +# `task_secrets` maps, so both deploy paths stay in sync automatically. + +data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { + secret_id = module.ooniapi_user.aws_access_key_id_arn +} + +data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { + secret_id = module.ooniapi_user.aws_secret_access_key_arn +} + +locals { + ooniapi_deploy_service_secrets = { + reverseproxy = { + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniprobe = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret_legacy.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value + } + oonirun = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + oonifindings = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + ooniauth = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string + AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string + } + oonimeasurements = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_test_url.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + } + } +} + +resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + name = "oonidevops/ooniapi/${each.key}/service_secrets" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id + secret_string = jsonencode(each.value) +} + +# Shared by every service's blue/green deploy job. The private key itself is +# generated and rotated out-of-band (see the deploy README); Terraform only +# owns the secret container, not its value. +resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { + name = "oonidevops/ooniapi/deploy_ssh_key" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { + secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id + secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" + + lifecycle { + ignore_changes = [secret_string] + } +} + resource "random_id" "artifact_id" { byte_length = 4 } @@ -557,6 +647,10 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniprobe" repo = "ooni/backend" branch_name = "master" @@ -569,6 +663,22 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + FASTPATH_URL = "http://fastpath.${local.environment}.ooni.io:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 3 # use a different one in prod + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -634,6 +744,10 @@ module "ooniapi_ooniprobe" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "reverseproxy" repo = "ooni/backend" branch_name = "master" @@ -646,6 +760,14 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + TARGET_URL = "https://backend-hel.ooni.org/" + } + secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -941,6 +1063,10 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonirun" repo = "ooni/backend" branch_name = "oonirun-v2-1" @@ -953,6 +1079,12 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -992,6 +1124,10 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonifindings" repo = "ooni/backend" branch_name = "master" @@ -1004,6 +1140,12 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1043,6 +1185,10 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniauth" repo = "ooni/backend" branch_name = "master" @@ -1055,6 +1201,27 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + AWS_REGION = var.aws_region + EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 + ADMIN_EMAILS = jsonencode([ + "maja@ooni.org", + "arturo@ooni.org", + "mehul@ooni.org", + "norbel@ooni.org", + "maria@ooni.org", + "admin+dev@ooni.org", + "luis@openobservatory.org", + "contact@openobservatory.org" + ]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1112,6 +1279,10 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonimeasurements" repo = "ooni/backend" branch_name = "master" @@ -1122,6 +1293,21 @@ module "ooniapi_oonimeasurements_deployer" { codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # it has to be a json-compliant array + OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + BASE_URL = "https://api.${local.environment}.ooni.io" + S3_BUCKET_NAME = "ooni-data-eu-fra-test" + VALKEY_URL = local.ooniapi_valkey_url + RATE_LIMITS = "10/minute;400000/day;200000/7day" + RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn + ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name } diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 2fb42ce7..dfa6de02 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -279,6 +279,95 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } +### Blue/green deploy secrets (Podman Quadlet on dedicated Hetzner hosts) +# +# These mirror the values already passed as `task_secrets` to the ECS task +# definitions above, but consolidated into one JSON blob per service so the +# "blue_green" deploy_mode CodeBuild job can fetch them in a single +# secretsmanager:GetSecretValue call and push each key as a Podman secret. +# Values are pulled from the same underlying data sources used by the ECS +# `task_secrets` maps, so both deploy paths stay in sync automatically. + +data "aws_secretsmanager_secret_version" "ooniapi_user_access_key_id" { + secret_id = module.ooniapi_user.aws_access_key_id_arn +} + +data "aws_secretsmanager_secret_version" "ooniapi_user_secret_access_key" { + secret_id = module.ooniapi_user.aws_secret_access_key_arn +} + +locals { + ooniapi_deploy_service_secrets = { + reverseproxy = { + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniprobe = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_write_url.value + ANONC_SECRET_KEY = data.aws_ssm_parameter.anonc_secret_key.value + } + oonirun = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.value + } + oonifindings = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + } + ooniauth = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + AWS_SECRET_ACCESS_KEY = data.aws_secretsmanager_secret_version.ooniapi_user_secret_access_key.secret_string + AWS_ACCESS_KEY_ID = data.aws_secretsmanager_secret_version.ooniapi_user_access_key_id.secret_string + } + oonimeasurements = { + POSTGRESQL_URL = data.aws_ssm_parameter.oonipg_url.value + JWT_ENCRYPTION_KEY = data.aws_ssm_parameter.jwt_secret.value + PROMETHEUS_METRICS_PASSWORD = data.aws_ssm_parameter.prometheus_metrics_password.value + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_oonimeasurements_url.value + ACCOUNT_ID_HASHING_KEY = data.aws_ssm_parameter.account_id_hashing_key.value + } + } +} + +resource "aws_secretsmanager_secret" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + name = "oonidevops/ooniapi/${each.key}/service_secrets" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_service_secrets" { + for_each = local.ooniapi_deploy_service_secrets + + secret_id = aws_secretsmanager_secret.ooniapi_deploy_service_secrets[each.key].id + secret_string = jsonencode(each.value) +} + +# Shared by every service's blue/green deploy job. The private key itself is +# generated and rotated out-of-band (see the deploy README); Terraform only +# owns the secret container, not its value. +resource "aws_secretsmanager_secret" "ooniapi_deploy_ssh_key" { + name = "oonidevops/ooniapi/deploy_ssh_key" + tags = local.tags +} + +resource "aws_secretsmanager_secret_version" "ooniapi_deploy_ssh_key" { + secret_id = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.id + secret_string = "REPLACE_ME: populate out-of-band with the \"deploy\" user's SSH private key" + + lifecycle { + ignore_changes = [secret_string] + } +} + module "geoip_bucket" { source = "../../modules/s3_bucket" @@ -433,6 +522,10 @@ module "ooni_th_droplet" { module "ooniapi_reverseproxy_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "reverseproxy" repo = "ooni/backend" branch_name = "master" @@ -445,6 +538,14 @@ module "ooniapi_reverseproxy_deployer" { ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + TARGET_URL = "https://backend-fsn.ooni.org/" + } + secrets = keys(local.ooniapi_deploy_service_secrets.reverseproxy) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["reverseproxy"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_reverseproxy" { @@ -857,6 +958,10 @@ EOF module "ooniapi_ooniprobe_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniprobe" repo = "ooni/backend" branch_name = "master" @@ -869,6 +974,23 @@ module "ooniapi_ooniprobe_deployer" { ecs_service_name = module.ooniapi_ooniprobe.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # hardcoded IP for fastpath2.prod.prod.ooni.io + FASTPATH_URL = "http://10.0.0.32:8472" + FASTPATH_URLS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8472"]) + FAILED_REPORTS_BUCKET = aws_s3_bucket.ooniprobe_failed_reports.bucket + COLLECTOR_ID = 4 # be sure this is different from dev + CONFIG_BUCKET = aws_s3_bucket.ooni_private_config_bucket.bucket + TOR_TARGETS = "tor_targets.json" + PSIPHON_CONFIG = "psiphon_config.json" + ANONC_MANIFEST_BUCKET = aws_s3_bucket.anoncred_manifests.bucket + ANONC_MANIFEST_FILE = "manifest.json" + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniprobe) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniprobe"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniprobe" { @@ -1027,6 +1149,10 @@ module "fastpath_builder" { module "ooniapi_oonirun_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonirun" repo = "ooni/backend" branch_name = "master" @@ -1039,6 +1165,12 @@ module "ooniapi_oonirun_deployer" { ecs_service_name = module.ooniapi_oonirun.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonirun) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonirun"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonirun" { @@ -1079,6 +1211,10 @@ module "ooniapi_oonirun" { module "ooniapi_oonifindings_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonifindings" repo = "ooni/backend" branch_name = "master" @@ -1091,6 +1227,12 @@ module "ooniapi_oonifindings_deployer" { ecs_service_name = module.ooniapi_oonifindings.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = {} + secrets = keys(local.ooniapi_deploy_service_secrets.oonifindings) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonifindings"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonifindings" { @@ -1131,6 +1273,10 @@ module "ooniapi_oonifindings" { module "ooniapi_ooniauth_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "ooniauth" repo = "ooni/backend" branch_name = "master" @@ -1143,6 +1289,27 @@ module "ooniapi_ooniauth_deployer" { ecs_service_name = module.ooniapi_ooniauth.ecs_service_name ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + AWS_REGION = var.aws_region + EMAIL_SOURCE_ADDRESS = module.ooniapi_user.email_address + SESSION_EXPIRY_DAYS = 2 + LOGIN_EXPIRY_DAYS = 7 + ADMIN_EMAILS = jsonencode([ + "maja@ooni.org", + "arturo@ooni.org", + "mehul@ooni.org", + "norbel@ooni.org", + "maria@ooni.org", + "admin+dev@ooni.org", + "luis@openobservatory.org", + "contact@openobservatory.org" + ]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.ooniauth) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["ooniauth"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_ooniauth" { @@ -1202,6 +1369,10 @@ module "ooniapi_ooniauth" { module "ooniapi_oonimeasurements_deployer" { source = "../../modules/ooniapi_service_deployer" + # Flip to "blue_green" to switch this service to the Podman Quadlet + # blue/green deploy on the dedicated Hetzner hosts. + deploy_mode = "ecs" + service_name = "oonimeasurements" repo = "ooni/backend" branch_name = "master" @@ -1215,6 +1386,21 @@ module "ooniapi_oonimeasurements_deployer" { ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name ecs_cluster_name = module.oonitier1plus_cluster.cluster_name # ecs_cluster_name = module.ooniapi_cluster.cluster_name + + # Pre-wired for the future flip to deploy_mode = "blue_green" + env_vars = { + # it has to be a json-compliant array + OTHER_COLLECTORS = jsonencode([for h in local.fastpath_hosts : "http://${h}:8475"]) + BASE_URL = "https://api.ooni.io" + S3_BUCKET_NAME = "ooni-data-eu-fra" + VALKEY_URL = local.ooniapi_valkey_url + RATE_LIMITS = "4000/hour;400000/day;200000/7day" + RATE_LIMITS_WHITELISTED_IPADDRS = jsonencode(["5.9.112.244"]) + RATE_LIMITS_UNMETERED_PAGES = jsonencode(["/metrics", "/health"]) + } + secrets = keys(local.ooniapi_deploy_service_secrets.oonimeasurements) + service_secrets_arn = aws_secretsmanager_secret.ooniapi_deploy_service_secrets["oonimeasurements"].arn + deploy_ssh_key_secret_arn = aws_secretsmanager_secret.ooniapi_deploy_ssh_key.arn } module "ooniapi_oonimeasurements" { From b09d13783ce5e8db7e8c550d7dc86ca075484a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 10 Jul 2026 21:52:25 +0200 Subject: [PATCH 3/3] Add ansible setup for ooniapi-gateway --- ansible/deploy-ooniapi-gateway.yml | 7 + .../roles/ooniapi_gateway/defaults/main.yml | 63 +++++++ .../roles/ooniapi_gateway/handlers/main.yml | 9 + ansible/roles/ooniapi_gateway/tasks/main.yml | 119 +++++++++++++ .../ooniapi_gateway/templates/gateway.conf.j2 | 167 ++++++++++++++++++ .../templates/sudoers-ooniapi-deploy.j2 | 23 +++ .../templates/buildspec_deploy.yml | 2 +- 7 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 ansible/deploy-ooniapi-gateway.yml create mode 100644 ansible/roles/ooniapi_gateway/defaults/main.yml create mode 100644 ansible/roles/ooniapi_gateway/handlers/main.yml create mode 100644 ansible/roles/ooniapi_gateway/tasks/main.yml create mode 100644 ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 create mode 100644 ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 diff --git a/ansible/deploy-ooniapi-gateway.yml b/ansible/deploy-ooniapi-gateway.yml new file mode 100644 index 00000000..f65c03a9 --- /dev/null +++ b/ansible/deploy-ooniapi-gateway.yml @@ -0,0 +1,7 @@ +--- +- name: Bootstrap the tier0 blue/green gateway on dedicated backend hosts + hosts: backend-fsn.ooni.org:backend-hel.ooni.org + become: true + roles: + - role: ooniapi_gateway + tags: ooniapi_gateway diff --git a/ansible/roles/ooniapi_gateway/defaults/main.yml b/ansible/roles/ooniapi_gateway/defaults/main.yml new file mode 100644 index 00000000..57ef60dd --- /dev/null +++ b/ansible/roles/ooniapi_gateway/defaults/main.yml @@ -0,0 +1,63 @@ +--- +# Domain the gateway vhost answers on for path-based routing. Mirrors the +# ALB's `ooniapi_frontend_main_domain_name` in +# tf/modules/ooniapi_frontend/main.tf (e.g. "api.prod.ooni.io"). +# No default: set explicitly per host in host_vars. +ooniapi_gateway_primary_domain: "" + +# Any additional hostnames that should hit the same path-routed vhost. +# Mirrors extra entries in `ooniapi_frontend_alternative_domains` that +# aren't handled elsewhere (e.g. "api.ooni.org" in prod). +ooniapi_gateway_extra_server_names: [] + +# Suffix used for the ALB's secondary, host-header-only routing rules, e.g. +# "oonirun.prod.ooni.io". Mirrors `direct_domain_suffix` in +# tf/modules/ooniapi_frontend/main.tf ("${stage}.ooni.io"). +# No default: set explicitly per host in host_vars. +ooniapi_gateway_direct_domain_suffix: "" + +# Name of the dehydrated cert (the directory under +# ooniapi_gateway_cert_path) that covers ooniapi_gateway_primary_domain, +# every ".{{ ooniapi_gateway_direct_domain_suffix }}" name, and +# ooniapi_gateway_extra_server_names as SANs. +# +# This role does NOT manage that cert -- issuing it means extending (or +# adding to) the `dehydrated` role's ssl_domains on this host, which in turn +# requires DNS for every one of those names to already resolve here. That's +# a DNS-cutover-time step, not a day-1 bootstrap step, so it's deliberately +# left manual. See the role README notes. +ooniapi_gateway_cert_name: "{{ ooniapi_gateway_primary_domain }}" +ooniapi_gateway_cert_path: /var/lib/dehydrated/certs/ + +# Flip to true only once ooniapi_gateway_cert_name actually exists on disk. +# Until then the vhost file is rendered but not installed, so this role is +# safe to run ahead of the real DNS cutover. +ooniapi_gateway_cert_ready: false + +# Services fronted by this gateway, and the two host ports their blue/green +# slots are bound to. host_port_a/host_port_b must match the values passed +# to that service's ooniapi_service_deployer module invocation in Terraform. +# +# Example: +# ooniapi_gateway_services: +# - name: reverseproxy +# host_port_a: 18001 +# host_port_b: 18002 +# - name: oonirun +# host_port_a: 18011 +# host_port_b: 18012 +ooniapi_gateway_services: [] + +# Cross-cloud passthrough for tier0 services not yet migrated off AWS +# (currently just testlists). Set to its public hostname to proxy those +# paths there until testlists itself is migrated; leave empty to omit +# those routes entirely. +ooniapi_gateway_testlists_upstream: "" + +# Podman network shared by every service's containers on this host. +ooniapi_gateway_network_name: ooniapi + +# SSH user the deploy CodeBuild job connects as. Must match +# `deploy_ssh_user` (default "deploy") in the ooniapi_service_deployer +# Terraform module. +ooniapi_gateway_deploy_user: deploy diff --git a/ansible/roles/ooniapi_gateway/handlers/main.yml b/ansible/roles/ooniapi_gateway/handlers/main.yml new file mode 100644 index 00000000..0fb3a40e --- /dev/null +++ b/ansible/roles/ooniapi_gateway/handlers/main.yml @@ -0,0 +1,9 @@ +--- +- name: test ooniapi gateway nginx config + ansible.builtin.command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: reload ooniapi gateway nginx + +- name: reload ooniapi gateway nginx + ansible.builtin.service: + name: nginx + state: reloaded diff --git a/ansible/roles/ooniapi_gateway/tasks/main.yml b/ansible/roles/ooniapi_gateway/tasks/main.yml new file mode 100644 index 00000000..94550e86 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/tasks/main.yml @@ -0,0 +1,119 @@ +--- +# One-time (and idempotent-on-rerun) bootstrap for a host that will run +# tier0 services via Podman Quadlet blue/green deploys, plus the nginx +# gateway vhost that fronts them. See the Terraform side of this in +# tf/modules/ooniapi_service_deployer (deploy_mode = "blue_green") and its +# templates/buildspec_deploy.yml, which is what actually performs each +# deploy -- this role only prepares the host for that job to run against. +# +# Runs as root (become: true at the play level). Podman state (networks, +# secrets) is created in root's namespace to match the root-owned systemd +# Quadlet units under /etc/containers/systemd/. + +- name: Ensure per-service state directories exist + ansible.builtin.file: + path: "/etc/ooniapi/{{ item.name }}" + state: directory + owner: root + group: root + mode: "0755" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + +- name: Check whether each service already has an active_slot marker + ansible.builtin.stat: + path: "/etc/ooniapi/{{ item.name }}/active_slot" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + register: ooniapi_gateway_active_slot_stat + +- name: Seed active_slot with "a" for services deploying for the first time + ansible.builtin.copy: + dest: "/etc/ooniapi/{{ item.item.name }}/active_slot" + content: "a" + owner: root + group: root + mode: "0644" + loop: "{{ ooniapi_gateway_active_slot_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: not item.stat.exists + +- name: Check whether each service already has an nginx upstream conf + ansible.builtin.stat: + path: "/etc/nginx/conf.d/{{ item.name }}-upstream.conf" + loop: "{{ ooniapi_gateway_services }}" + loop_control: + label: "{{ item.name }}" + register: ooniapi_gateway_upstream_stat + +# Seeded with slot "a" up / "b" down, matching active_slot's "a" default +# above, so the very first `nginx -t` (run when installing the gateway +# vhost below) succeeds before that service's first real deploy has run. +- name: Seed a placeholder upstream conf for services deploying for the first time + ansible.builtin.copy: + dest: "/etc/nginx/conf.d/{{ item.item.name }}-upstream.conf" + content: | + upstream {{ item.item.name }} { + server 127.0.0.1:{{ item.item.host_port_a }}; + server 127.0.0.1:{{ item.item.host_port_b }} down; + } + owner: root + group: root + mode: "0644" + loop: "{{ ooniapi_gateway_upstream_stat.results }}" + loop_control: + label: "{{ item.item.name }}" + when: not item.stat.exists + notify: reload ooniapi gateway nginx + +- name: Check whether the podman network already exists + ansible.builtin.command: podman network exists {{ ooniapi_gateway_network_name }} + register: ooniapi_gateway_network_check + changed_when: false + failed_when: false + +- name: Create the shared podman network + ansible.builtin.command: podman network create {{ ooniapi_gateway_network_name }} + when: ooniapi_gateway_network_check.rc != 0 + +- name: Ensure sudoers.d directory exists + ansible.builtin.file: + path: /etc/sudoers.d + state: directory + owner: root + group: root + +- name: Install scoped sudoers rule for the deploy user + ansible.builtin.template: + src: sudoers-ooniapi-deploy.j2 + dest: /etc/sudoers.d/90-ooniapi-deploy + owner: root + group: root + mode: "0440" + validate: "visudo -cf %s" + +- name: Render the gateway vhost + ansible.builtin.template: + src: gateway.conf.j2 + dest: /etc/nginx/conf.d/ooniapi-gateway.conf.pending + owner: root + group: root + mode: "0644" + +# Split into "render" + "install" so a bad template is visible +# (.conf.pending) without ever being loaded by nginx, and so the vhost is +# never installed at all until ooniapi_gateway_cert_ready is true -- see +# defaults/main.yml for why that's a separate, deliberate step. +- name: Install the gateway vhost now that its cert is ready + ansible.builtin.copy: + remote_src: true + src: /etc/nginx/conf.d/ooniapi-gateway.conf.pending + dest: /etc/nginx/conf.d/ooniapi-gateway.conf + owner: root + group: root + mode: "0644" + when: ooniapi_gateway_cert_ready + notify: reload ooniapi gateway nginx diff --git a/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 new file mode 100644 index 00000000..e87e46ef --- /dev/null +++ b/ansible/roles/ooniapi_gateway/templates/gateway.conf.j2 @@ -0,0 +1,167 @@ +# Managed by ansible - roles/ooniapi_gateway/templates/gateway.conf.j2 +# +# Single "gateway" vhost for the OONI tier0 API on this dedicated host, +# mirroring (1:1, by design) the AWS ALB listener rules in +# tf/modules/ooniapi_frontend/main.tf. Only rules for services actually +# running blue/green here ({{ ooniapi_gateway_services | map(attribute='name') | join(', ') }}) +# are handled; everything unmatched falls through to "reverseproxy", exactly +# like the ALB's default_action. +# +# Each service's own `-upstream.conf` (the `upstream { +# ... }` block referenced below via proxy_pass) is dropped into this same +# conf.d directory by that service's blue/green deploy job, not by this +# role -- see ooniapi_service_deployer's buildspec_deploy.yml. A service's +# location block here will fail nginx -t until that service has deployed at +# least once. + +{% set service_names = ooniapi_gateway_services | map(attribute='name') | list %} + +# anonymize ipaddr (same scheme as the legacy ooni-api vhost) +map $remote_addr $ooniapi_gateway_remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +log_format ooniapi_gateway_fmt '$ooniapi_gateway_remote_addr_anon [$time_local] ' + '"$request" $status $body_bytes_sent rt:$request_time "$http_referer" "$http_user_agent"'; + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ ([ooniapi_gateway_primary_domain] + ooniapi_gateway_extra_server_names) | join(' ') }}; + + access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + + client_max_body_size 200M; # for measurement POST, matches the legacy API vhost + + ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; + ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; + ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem; + + # https://ssl-config.mozilla.org/#server=nginx&config=intermediate -- kept + # inline (rather than shared include) so this role has no dependency on + # how nginx itself got installed on this host. + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + ssl_stapling on; + ssl_stapling_verify on; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + +{% if 'ooniauth' in service_names %} + ## --- ooniauth --- (ALB priority 108) + location ^~ /api/v2/ooniauth/ { proxy_pass http://ooniauth; } + location = /api/v1/user_register { proxy_pass http://ooniauth; } + location = /api/v1/user_login { proxy_pass http://ooniauth; } + location = /api/v1/user_refresh_token { proxy_pass http://ooniauth; } + location = /api/_/account_metadata { proxy_pass http://ooniauth; } +{% endif %} + +{% if 'oonirun' in service_names %} + ## --- oonirun --- (ALB priority 110) + location ^~ /api/v2/oonirun/ { proxy_pass http://oonirun; } +{% endif %} + +{% if 'ooniprobe' in service_names %} + ## --- ooniprobe --- (ALB priorities 120-123) + location ^~ /api/v2/ooniprobe/ { proxy_pass http://ooniprobe; } + location = /api/v1/login { proxy_pass http://ooniprobe; } + location = /api/v1/register { proxy_pass http://ooniprobe; } + location ^~ /api/v1/update/ { proxy_pass http://ooniprobe; } + location ^~ /api/v1/check-in { proxy_pass http://ooniprobe; } + location ^~ /api/v1/test-helpers { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/urls { proxy_pass http://ooniprobe; } + location ^~ /report { proxy_pass http://ooniprobe; } + location = /api/_/show_countries_prioritization { proxy_pass http://ooniprobe; } + location = /api/_/debug_prioritization { proxy_pass http://ooniprobe; } + location ^~ /api/v1/manifest { proxy_pass http://ooniprobe; } + location ^~ /api/v1/sign_credential { proxy_pass http://ooniprobe; } + location ^~ /api/v1/submit_measurement { proxy_pass http://ooniprobe; } + location ^~ /bouncer/net-tests { proxy_pass http://ooniprobe; } + location ^~ /api/v1/geolookup { proxy_pass http://ooniprobe; } + location ^~ /api/v1/collectors { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/tor-targets { proxy_pass http://ooniprobe; } + location = /api/v1/test-list/psiphon-config { proxy_pass http://ooniprobe; } +{% endif %} + +{% if 'oonifindings' in service_names %} + ## --- oonifindings --- (ALB priority 130) + location ^~ /api/v1/incidents/ { proxy_pass http://oonifindings; } +{% endif %} + +{% if 'oonimeasurements' in service_names %} + ## --- oonimeasurements --- (ALB priorities 140-143) + location ^~ /api/v1/measurements/ { proxy_pass http://oonimeasurements; } + location = /api/v1/raw_measurement { proxy_pass http://oonimeasurements; } + location = /api/v1/measurement_meta { proxy_pass http://oonimeasurements; } + location = /api/v1/measurements { proxy_pass http://oonimeasurements; } + location = /api/v1/torsf_stats { proxy_pass http://oonimeasurements; } + location = /api/v1/aggregation { proxy_pass http://oonimeasurements; } + location ^~ /api/v1/aggregation/ { proxy_pass http://oonimeasurements; } + location = /api/v1/observations { proxy_pass http://oonimeasurements; } + location = /api/v1/analysis { proxy_pass http://oonimeasurements; } + location = /api/v1/detector/changepoints { proxy_pass http://oonimeasurements; } +{% endif %} + +{% if ooniapi_gateway_testlists_upstream %} + ## --- testlists (ALB priority 144; not yet migrated off AWS, proxied + ## cross-cloud until it is) --- + location ^~ /api/_/url-submission/test-list/ { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/_/url-priorities/list { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/_/url-priorities/update { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/v1/url-submission/submit { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } + location = /api/v1/url-submission/update-url { proxy_pass https://{{ ooniapi_gateway_testlists_upstream }}; } +{% endif %} + + ## --- default: everything else goes to reverseproxy, same as the ALB's + ## default_action. Covers legacy/unmigrated paths (oonith is NOT among + ## them -- it's routed by host_header on *.th.ooni.org, a hostname this + ## vhost doesn't answer for, so it never reaches here either way). --- + location / { + proxy_pass http://reverseproxy; + } +} + +{% for service in service_names %} +{% if service != 'reverseproxy' %} +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name {{ service }}.{{ ooniapi_gateway_direct_domain_suffix }}; + + access_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info ooniapi_gateway_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi-gateway,severity=info; + + ssl_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/fullchain.pem; + ssl_certificate_key {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/privkey.pem; + ssl_trusted_certificate {{ ooniapi_gateway_cert_path }}{{ ooniapi_gateway_cert_name }}/chain.pem; + ssl_protocols TLSv1.2 TLSv1.3; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + location / { + proxy_pass http://{{ service }}; + } +} +{% endif %} +{% endfor %} diff --git a/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 new file mode 100644 index 00000000..d78a8d27 --- /dev/null +++ b/ansible/roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 @@ -0,0 +1,23 @@ +# Managed by ansible - roles/ooniapi_gateway/templates/sudoers-ooniapi-deploy.j2 +# +# Scoped to exactly the remote commands issued by +# tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml over +# SSH as {{ ooniapi_gateway_deploy_user }}. Intentionally narrower than the +# blanket "NOPASSWD:ALL" pattern used for the deploy user in other roles +# (e.g. roles/anonc) -- this user is driven by an automated CI job over the +# network, not a human operator. +# +# Caveat: sudoers glob matching (fnmatch without FNM_PATHNAME) lets "*" +# match "/". The mv sources below are always files this same job just +# scp'd into /tmp itself, so this is a theoretical rather than practical +# widening of scope, but worth knowing if you're auditing this file. + +Cmnd_Alias OONIAPI_DEPLOY_MV_UNIT = /usr/bin/mv /tmp/*.container /etc/containers/systemd/*.container +Cmnd_Alias OONIAPI_DEPLOY_MV_UPSTREAM = /usr/bin/mv /tmp/*-upstream.conf /etc/nginx/conf.d/*-upstream.conf +Cmnd_Alias OONIAPI_DEPLOY_SYSTEMCTL = /usr/bin/systemctl daemon-reload, /usr/bin/systemctl restart *-a.service, /usr/bin/systemctl restart *-b.service, /usr/bin/systemctl reload nginx +Cmnd_Alias OONIAPI_DEPLOY_NGINX_TEST = /usr/sbin/nginx -t +Cmnd_Alias OONIAPI_DEPLOY_MARKER = /usr/bin/tee /etc/ooniapi/*/active_slot +Cmnd_Alias OONIAPI_DEPLOY_PODMAN_SECRET = /usr/bin/podman secret create --replace * - + +{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_MV_UNIT, OONIAPI_DEPLOY_MV_UPSTREAM, OONIAPI_DEPLOY_SYSTEMCTL, OONIAPI_DEPLOY_NGINX_TEST, OONIAPI_DEPLOY_MARKER +{{ ooniapi_gateway_deploy_user }} ALL=(root) NOPASSWD: OONIAPI_DEPLOY_PODMAN_SECRET diff --git a/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml b/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml index 596aad30..1ea3968e 100644 --- a/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml +++ b/tf/modules/ooniapi_service_deployer/templates/buildspec_deploy.yml @@ -49,7 +49,7 @@ phases: # stdin, never interpolated into the remote command string) for KEY in $(jq -r 'keys[]' /tmp/service_secrets.json); do VALUE=$(jq -r --arg k "$KEY" '.[$k]' /tmp/service_secrets.json) - printf '%s' "$VALUE" | ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "podman secret create --replace ${KEY} -" + printf '%s' "$VALUE" | ssh $SSH_OPTS "${DEPLOY_SSH_USER}@${HOST}" "sudo podman secret create --replace ${KEY} -" done # d. reload systemd and (re)start only the target slot