docker-compose.swarm.ha.yml

# Obiente Cloud - High Availability Docker Swarm Configuration
# This setup supports distributed deployments across multiple nodes



# This file requires docker-compose.base.yml to be merged before use
# Usage: cat docker-compose.base.yml docker-compose.swarm.ha.yml | docker stack deploy -c - obiente

services:
  # ============================================================================
  # DISTRIBUTED DATABASE CLUSTER (PostgreSQL + Patroni)
  # ============================================================================

  # etcd - Distributed consensus for Patroni
  etcd:
    image: quay.io/coreos/etcd:v3.5.12
    command:
      - /usr/local/bin/etcd
      - --name=etcd
      - --initial-advertise-peer-urls=http://etcd:2380
      - --listen-peer-urls=http://0.0.0.0:2380
      - --listen-client-urls=http://0.0.0.0:2379
      - --advertise-client-urls=http://etcd:2379
      - --initial-cluster=etcd=http://etcd:2380
      - --initial-cluster-state=new
      - --initial-cluster-token=obiente-cluster
    deploy:
      replicas: 3
      placement:
        max_replicas_per_node: 1
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
    volumes:
      - etcd_data:/etcd-data
    networks:
      - obiente-network

  # HAProxy - PostgreSQL connection pooler and load balancer
  pgpool:
    image: bitnami/pgpool:4
    environment:
      PGPOOL_BACKEND_NODES: 0:patroni-1:5432,1:patroni-2:5432,2:patroni-3:5432
      PGPOOL_SR_CHECK_USER: ${POSTGRES_USER:-obiente_postgres}
      PGPOOL_SR_CHECK_PASSWORD: ${POSTGRES_PASSWORD:-obiente_postgres}
      PGPOOL_ENABLE_LDAP: "no"
      PGPOOL_POSTGRES_USERNAME: ${POSTGRES_USER:-obiente_postgres}
      PGPOOL_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-obiente_postgres}
      PGPOOL_ADMIN_USERNAME: admin
      PGPOOL_ADMIN_PASSWORD: ${PGPOOL_ADMIN_PASSWORD:-adminpassword}
      PGPOOL_ENABLE_LOAD_BALANCING: "yes"
      PGPOOL_MAX_POOL: 100
    deploy:
      replicas: 2
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
    healthcheck:
      test: ["CMD", "/opt/bitnami/scripts/pgpool/healthcheck.sh"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - obiente-network

  # Patroni PostgreSQL - Node 1 (Leader)
  patroni-1:
    image: patroni/patroni:3.2.2
    hostname: patroni-1
    environment:
      PATRONI_NAME: patroni-1
      PATRONI_SCOPE: obiente-cluster
      PATRONI_ETCD3_HOSTS: etcd:2379
      PATRONI_RESTAPI_CONNECT_ADDRESS: patroni-1:8008
      PATRONI_RESTAPI_LISTEN: 0.0.0.0:8008
      PATRONI_POSTGRESQL_CONNECT_ADDRESS: patroni-1:5432
      PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5432
      PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data
      PATRONI_POSTGRESQL_PGPASS: /tmp/pgpass
      PATRONI_SUPERUSER_USERNAME: ${POSTGRES_USER:-obiente_postgres}
      PATRONI_SUPERUSER_PASSWORD: ${POSTGRES_PASSWORD:-obiente_postgres}
      PATRONI_REPLICATION_USERNAME: replicator
      PATRONI_REPLICATION_PASSWORD: ${REPLICATION_PASSWORD:-replicator-password}
      PATRONI_admin_PASSWORD: ${PATRONI_ADMIN_PASSWORD:-admin}
      PATRONI_admin_OPTIONS: "createdb,createrole"
    volumes:
      - patroni_1_data:/var/lib/postgresql/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.postgres.replica == 1
      resources:
        limits:
          cpus: "4"
          memory: 4G
        reservations:
          cpus: "2"
          memory: 2G
    networks:
      - obiente-network

  # Patroni PostgreSQL - Node 2 (Replica)
  patroni-2:
    image: patroni/patroni:3.2.2
    hostname: patroni-2
    environment:
      PATRONI_NAME: patroni-2
      PATRONI_SCOPE: obiente-cluster
      PATRONI_ETCD3_HOSTS: etcd:2379
      PATRONI_RESTAPI_CONNECT_ADDRESS: patroni-2:8008
      PATRONI_RESTAPI_LISTEN: 0.0.0.0:8008
      PATRONI_POSTGRESQL_CONNECT_ADDRESS: patroni-2:5432
      PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5432
      PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data
      PATRONI_POSTGRESQL_PGPASS: /tmp/pgpass
      PATRONI_SUPERUSER_USERNAME: ${POSTGRES_USER:-obiente_postgres}
      PATRONI_SUPERUSER_PASSWORD: ${POSTGRES_PASSWORD:-obiente_postgres}
      PATRONI_REPLICATION_USERNAME: replicator
      PATRONI_REPLICATION_PASSWORD: ${REPLICATION_PASSWORD:-replicator-password}
      PATRONI_admin_PASSWORD: ${PATRONI_ADMIN_PASSWORD:-admin}
      PATRONI_admin_OPTIONS: "createdb,createrole"
    volumes:
      - patroni_2_data:/var/lib/postgresql/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.postgres.replica == 2
      resources:
        limits:
          cpus: "4"
          memory: 4G
        reservations:
          cpus: "2"
          memory: 2G
    networks:
      - obiente-network

  # Patroni PostgreSQL - Node 3 (Replica)
  patroni-3:
    image: patroni/patroni:3.2.2
    hostname: patroni-3
    environment:
      PATRONI_NAME: patroni-3
      PATRONI_SCOPE: obiente-cluster
      PATRONI_ETCD3_HOSTS: etcd:2379
      PATRONI_RESTAPI_CONNECT_ADDRESS: patroni-3:8008
      PATRONI_RESTAPI_LISTEN: 0.0.0.0:8008
      PATRONI_POSTGRESQL_CONNECT_ADDRESS: patroni-3:5432
      PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5432
      PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data
      PATRONI_POSTGRESQL_PGPASS: /tmp/pgpass
      PATRONI_SUPERUSER_USERNAME: ${POSTGRES_USER:-obiente_postgres}
      PATRONI_SUPERUSER_PASSWORD: ${POSTGRES_PASSWORD:-obiente_postgres}
      PATRONI_REPLICATION_USERNAME: replicator
      PATRONI_REPLICATION_PASSWORD: ${REPLICATION_PASSWORD:-replicator-password}
      PATRONI_admin_PASSWORD: ${PATRONI_ADMIN_PASSWORD:-admin}
      PATRONI_admin_OPTIONS: "createdb,createrole"
    volumes:
      - patroni_3_data:/var/lib/postgresql/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.postgres.replica == 3
      resources:
        limits:
          cpus: "4"
          memory: 4G
        reservations:
          cpus: "2"
          memory: 2G
    networks:
      - obiente-network

  # ============================================================================
  # METRICS DATABASE CLUSTER (TimescaleDB + Patroni)
  # ============================================================================

  # PgPool for TimescaleDB - Connection pooler and load balancer
  metrics-pgpool:
    image: bitnami/pgpool:4
    environment:
      PGPOOL_BACKEND_NODES: 0:metrics-patroni-1:5432,1:metrics-patroni-2:5432,2:metrics-patroni-3:5432
      PGPOOL_SR_CHECK_USER: ${METRICS_DB_USER:-obiente_postgres}
      PGPOOL_SR_CHECK_PASSWORD: ${METRICS_DB_PASSWORD:-obiente_postgres}
      PGPOOL_ENABLE_LDAP: "no"
      PGPOOL_POSTGRES_USERNAME: ${METRICS_DB_USER:-obiente_postgres}
      PGPOOL_POSTGRES_PASSWORD: ${METRICS_DB_PASSWORD:-obiente_postgres}
      PGPOOL_ADMIN_USERNAME: admin
      PGPOOL_ADMIN_PASSWORD: ${METRICS_PGPOOL_ADMIN_PASSWORD:-${PGPOOL_ADMIN_PASSWORD:-adminpassword}}
      PGPOOL_ENABLE_LOAD_BALANCING: "yes"
      PGPOOL_MAX_POOL: 100
    deploy:
      replicas: 2
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
    healthcheck:
      test: ["CMD", "/opt/bitnami/scripts/pgpool/healthcheck.sh"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - obiente-network

  # Patroni TimescaleDB - Node 1 (Leader)
  metrics-patroni-1:
    image: timescale/timescaledb:latest-pg16
    hostname: metrics-patroni-1
    environment:
      PATRONI_NAME: metrics-patroni-1
      PATRONI_SCOPE: metrics-cluster
      PATRONI_ETCD3_HOSTS: etcd:2379
      PATRONI_RESTAPI_CONNECT_ADDRESS: metrics-patroni-1:8008
      PATRONI_RESTAPI_LISTEN: 0.0.0.0:8008
      PATRONI_POSTGRESQL_CONNECT_ADDRESS: metrics-patroni-1:5432
      PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5432
      PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data
      PATRONI_POSTGRESQL_PGPASS: /tmp/pgpass
      PATRONI_SUPERUSER_USERNAME: ${METRICS_DB_USER:-obiente_postgres}
      PATRONI_SUPERUSER_PASSWORD: ${METRICS_DB_PASSWORD:-obiente_postgres}
      PATRONI_REPLICATION_USERNAME: metrics-replicator
      PATRONI_REPLICATION_PASSWORD: ${METRICS_REPLICATION_PASSWORD:-${REPLICATION_PASSWORD:-metrics-replicator-password}}
      PATRONI_admin_PASSWORD: ${METRICS_PATRONI_ADMIN_PASSWORD:-${PATRONI_ADMIN_PASSWORD:-admin}}
      PATRONI_admin_OPTIONS: "createdb,createrole"
    volumes:
      - metrics_patroni_1_data:/var/lib/postgresql/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.metrics.replica == 1
      resources:
        limits:
          cpus: "4"
          memory: 4G
        reservations:
          cpus: "2"
          memory: 4G
    networks:
      - obiente-network

  # Patroni TimescaleDB - Node 2 (Replica)
  metrics-patroni-2:
    image: timescale/timescaledb:latest-pg16
    hostname: metrics-patroni-2
    environment:
      PATRONI_NAME: metrics-patroni-2
      PATRONI_SCOPE: metrics-cluster
      PATRONI_ETCD3_HOSTS: etcd:2379
      PATRONI_RESTAPI_CONNECT_ADDRESS: metrics-patroni-2:8008
      PATRONI_RESTAPI_LISTEN: 0.0.0.0:8008
      PATRONI_POSTGRESQL_CONNECT_ADDRESS: metrics-patroni-2:5432
      PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5432
      PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data
      PATRONI_POSTGRESQL_PGPASS: /tmp/pgpass
      PATRONI_SUPERUSER_USERNAME: ${METRICS_DB_USER:-obiente_postgres}
      PATRONI_SUPERUSER_PASSWORD: ${METRICS_DB_PASSWORD:-obiente_postgres}
      PATRONI_REPLICATION_USERNAME: metrics-replicator
      PATRONI_REPLICATION_PASSWORD: ${METRICS_REPLICATION_PASSWORD:-${REPLICATION_PASSWORD:-metrics-replicator-password}}
      PATRONI_admin_PASSWORD: ${METRICS_PATRONI_ADMIN_PASSWORD:-${PATRONI_ADMIN_PASSWORD:-admin}}
      PATRONI_admin_OPTIONS: "createdb,createrole"
    volumes:
      - metrics_patroni_2_data:/var/lib/postgresql/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.metrics.replica == 2
      resources:
        limits:
          cpus: "4"
          memory: 4G
        reservations:
          cpus: "2"
          memory: 4G
    networks:
      - obiente-network

  # Patroni TimescaleDB - Node 3 (Replica)
  metrics-patroni-3:
    image: timescale/timescaledb:latest-pg16
    hostname: metrics-patroni-3
    environment:
      PATRONI_NAME: metrics-patroni-3
      PATRONI_SCOPE: metrics-cluster
      PATRONI_ETCD3_HOSTS: etcd:2379
      PATRONI_RESTAPI_CONNECT_ADDRESS: metrics-patroni-3:8008
      PATRONI_RESTAPI_LISTEN: 0.0.0.0:8008
      PATRONI_POSTGRESQL_CONNECT_ADDRESS: metrics-patroni-3:5432
      PATRONI_POSTGRESQL_LISTEN: 0.0.0.0:5432
      PATRONI_POSTGRESQL_DATA_DIR: /var/lib/postgresql/data
      PATRONI_POSTGRESQL_PGPASS: /tmp/pgpass
      PATRONI_SUPERUSER_USERNAME: ${METRICS_DB_USER:-obiente_postgres}
      PATRONI_SUPERUSER_PASSWORD: ${METRICS_DB_PASSWORD:-obiente_postgres}
      PATRONI_REPLICATION_USERNAME: metrics-replicator
      PATRONI_REPLICATION_PASSWORD: ${METRICS_REPLICATION_PASSWORD:-${REPLICATION_PASSWORD:-metrics-replicator-password}}
      PATRONI_admin_PASSWORD: ${METRICS_PATRONI_ADMIN_PASSWORD:-${PATRONI_ADMIN_PASSWORD:-admin}}
      PATRONI_admin_OPTIONS: "createdb,createrole"
    volumes:
      - metrics_patroni_3_data:/var/lib/postgresql/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.metrics.replica == 3
      resources:
        limits:
          cpus: "4"
          memory: 4G
        reservations:
          cpus: "2"
          memory: 4G
    networks:
      - obiente-network

  # ============================================================================
  # REDIS CLUSTER (for caching, sessions, job queues)
  # ============================================================================

  redis-1:
    image: redis:8-alpine
    command: >
      redis-server
      --appendonly yes
      --cluster-enabled yes
      --cluster-config-file nodes.conf
      --cluster-node-timeout 5000
      --port 6379
    volumes:
      - redis_1_data:/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.redis == 1
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 512M
    networks:
      - obiente-network

  redis-2:
    image: redis:8-alpine
    command: >
      redis-server
      --appendonly yes
      --cluster-enabled yes
      --cluster-config-file nodes.conf
      --cluster-node-timeout 5000
      --port 6379
    volumes:
      - redis_2_data:/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.redis == 2
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 512M
    networks:
      - obiente-network

  redis-3:
    image: redis:8-alpine
    command: >
      redis-server
      --appendonly yes
      --cluster-enabled yes
      --cluster-config-file nodes.conf
      --cluster-node-timeout 5000
      --port 6379
    volumes:
      - redis_3_data:/data
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.labels.redis == 3
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 512M
    networks:
      - obiente-network

  # ============================================================================
  # SERVICE DISCOVERY & ROUTING
  # ============================================================================

  # Traefik - Dynamic reverse proxy with service discovery
  traefik:
    image: traefik:v3.6.1
    command:
      - --api.dashboard=true
      - --api.insecure=false  # Use secure API (HTTPS only)
      - --providers.swarm=true  # Use Swarm provider for Docker Swarm mode (v3.0)
      - --providers.swarm.exposedbydefault=false
      - --providers.swarm.network=__STACK_NAME___obiente-network
      - --providers.swarm.watch=true  # Watch for service changes
      - --providers.swarm.refreshSeconds=5  # Poll Docker Swarm every 5 seconds for faster service discovery
      - --providers.swarm.constraints=Label(`cloud.obiente.traefik`,`true`)  # Only discover services with this label
      # Timeout configuration to fail faster on unreachable backends
      # dialTimeout: time to establish connection (increased to 10s to handle network delays)
      - --serversTransport.forwardingTimeouts.dialTimeout=10s
      # responseHeaderTimeout: time to wait for response headers
      - --serversTransport.forwardingTimeouts.responseHeaderTimeout=15s
      # idleConnTimeout: time to keep idle connections open
      - --serversTransport.forwardingTimeouts.idleConnTimeout=90s
      - --providers.etcd.endpoints=etcd:2379
      - --providers.etcd.rootkey=traefik
      - --entryPoints.web.address=:80
      - --entryPoints.web.forwardedHeaders.trustedIPs=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
      - --entryPoints.websecure.address=:443
      - --entryPoints.websecure.forwardedHeaders.trustedIPs=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
      - --entryPoints.deployments.address=:8000
      - --entryPoints.deployments.forwardedHeaders.trustedIPs=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
      - --entryPoints.ssh.address=:${SSH_PROXY_PORT:-2222}
      - --entryPoints.ssh.proxyProtocol.trustedIPs=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
      # Database TCP entrypoints
      - --entryPoints.postgres.address=:5432
      - --entryPoints.mysql.address=:3306
      - --entryPoints.mongodb.address=:27017
      - --entryPoints.redis.address=:6379
      - --certificatesresolvers.letsencrypt.acme.email=${ACME_EMAIL}
      - --certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json
      - --certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web
      # Use Let's Encrypt staging environment if ACME_CA_SERVER is set to staging URL
      # Set ACME_CA_SERVER=https://acme-staging-v02.api.letsencrypt.org/directory to avoid rate limits during testing
      # Default: production (https://acme-v02.api.letsencrypt.org/directory)
      - --certificatesresolvers.letsencrypt.acme.caserver=${ACME_CA_SERVER:-https://acme-v02.api.letsencrypt.org/directory}
      - --metrics.prometheus=true
      - --accesslog=true
      - --log.level=INFO
    ports:
      - target: 80
        published: 80
        protocol: tcp
        mode: host  # host mode preserves real client IPs (ingress mode causes SNAT)
      - target: 443
        published: 443
        protocol: tcp
        mode: host  # host mode preserves real client IPs (ingress mode causes SNAT)
      - target: 8000
        published: 8000
        protocol: tcp
        mode: host  # host mode preserves real client IPs (ingress mode causes SNAT)
      - target: 8080
        published: 8080
        protocol: tcp
        mode: host  # Traefik dashboard accessible on port 8080
      - target: ${SSH_PROXY_PORT:-2222}
        published: ${SSH_PROXY_PORT:-2222}
        protocol: tcp
        mode: host  # host mode preserves real client IPs for SSH connections
      # Database ports
      - target: 5432
        published: 5432
        protocol: tcp
        mode: ingress
      - target: 3306
        published: 3306
        protocol: tcp
        mode: ingress
      - target: 27017
        published: 27017
        protocol: tcp
        mode: ingress
      - target: 6379
        published: 6379
        protocol: tcp
        mode: ingress
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - traefik_letsencrypt:/letsencrypt
    depends_on:
      - etcd
    deploy:
      mode: global  # Run on all nodes for HA and DNS to resolve properly
      update_config:
        parallelism: 1
        order: start-first
      placement:
        constraints:
          - node.labels.traefik.exclude != true  # Exclude nodes labeled with traefik.exclude=true
      resources:
        limits:
          cpus: "2"
          memory: 512M
        reservations:
          cpus: "1"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.traefik.rule=Host(`traefik.${DOMAIN}`)"
        - "traefik.http.routers.traefik.entrypoints=websecure"
        - "traefik.http.routers.traefik.tls.certresolver=letsencrypt"
        - "traefik.http.routers.traefik.service=api@internal"
        - "traefik.http.services.traefik.loadbalancer.server.port=8080"
        # API routing
        - "traefik.http.routers.api.rule=Host(`api.${DOMAIN}`)"
        - "traefik.http.routers.api.entrypoints=websecure"
        - "traefik.http.routers.api.tls.certresolver=letsencrypt"
        # Note: Forwarded headers are configured at entrypoint level in Traefik command
        # This ensures real client IPs are forwarded from Docker network IPs
        - "traefik.http.services.api.loadbalancer.server.port=3001"
    networks:
      - obiente-network

  # ============================================================================
  # MONITORING & OBSERVABILITY
  # ============================================================================

  # Prometheus - Metrics collection
  prometheus:
    image: prom/prometheus:latest
    user: "0:0"  # Run as root to access Docker socket for Swarm service discovery
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.path=/prometheus"
      - "--storage.tsdb.retention.time=15d"
    volumes:
      - prometheus_data:/prometheus
      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro  # Required for Docker Swarm service discovery
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      resources:
        limits:
          cpus: "2"
          memory: 2G
        reservations:
          cpus: "1"
          memory: 512M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.prometheus.rule=Host(`prometheus.${DOMAIN}`)"
        - "traefik.http.routers.prometheus.entrypoints=websecure"
        - "traefik.http.routers.prometheus.tls.certresolver=letsencrypt"
        - "traefik.http.services.prometheus.loadbalancer.server.port=9090"
    networks:
      - obiente-network

  # Grafana - Visualization
  grafana:
    image: grafana/grafana:latest
    user: "0:0"  # Run as root for entrypoint script to write provisioning files
    entrypoint: ["/bin/sh", "/etc/grafana/entrypoint.sh"]
    command: ["/run.sh"]
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin}
      GF_SERVER_ROOT_URL: https://grafana.${DOMAIN}
      # Disable strict provisioning validation to avoid conflicts in HA setup
      GF_PROVISIONING_DATASOURCES_ALLOW_DELETION: "true"
      # Allow unsigned plugins (for lokiexplore-app and other community plugins)
      GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: "grafana-lokiexplore-app"
      # Pass database credentials as environment variables for datasource provisioning
      POSTGRES_USER: ${POSTGRES_USER:-obiente_postgres}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-obiente_postgres}
      METRICS_DB_USER: ${METRICS_DB_USER:-obiente_postgres}
      METRICS_DB_PASSWORD: ${METRICS_DB_PASSWORD:-obiente_postgres}
      # Datasource host configuration (for HA, use pgpool service names)
      GRAFANA_POSTGRES_HOST: ${GRAFANA_POSTGRES_HOST:-pgpool}
      GRAFANA_METRICS_DB_HOST: ${GRAFANA_METRICS_DB_HOST:-metrics-pgpool}
      # Alerting configuration
      ALERT_EMAIL: ${ALERT_EMAIL:-admin@example.com}
      # SMTP configuration for Grafana email notifications (only set if SMTP_HOST is configured)
      # These will be conditionally set by the entrypoint script
      SMTP_HOST: ${SMTP_HOST:-}
      SMTP_PORT: ${SMTP_PORT:-587}
      SMTP_USERNAME: ${SMTP_USERNAME:-}
      SMTP_PASSWORD: ${SMTP_PASSWORD:-}
      SMTP_FROM_ADDRESS: ${SMTP_FROM_ADDRESS:-}
      SMTP_FROM_NAME: ${SMTP_FROM_NAME:-Grafana}
      SMTP_SKIP_TLS_VERIFY: ${SMTP_SKIP_TLS_VERIFY:-false}
      SMTP_USE_STARTTLS: ${SMTP_USE_STARTTLS:-true}
    volumes:
      - grafana_data:/var/lib/grafana
      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:rw
      - ./monitoring/grafana/dashboards:/etc/grafana/dashboards:ro
      - ./monitoring/grafana/entrypoint.sh:/etc/grafana/entrypoint.sh:ro
    depends_on:
      - prometheus
      - pgpool
      - metrics-pgpool
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.grafana.rule=Host(`grafana.${DOMAIN}`)"
        - "traefik.http.routers.grafana.entrypoints=web"
        - "traefik.http.routers.grafana.service=grafana"
        - "traefik.http.services.grafana.loadbalancer.server.port=3000"
        # Route Grafana on HTTPS for production access
        - "traefik.http.routers.grafana-secure.rule=Host(`grafana.${DOMAIN}`)"
        - "traefik.http.routers.grafana-secure.entrypoints=websecure"
        - "traefik.http.routers.grafana-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.grafana-secure.service=grafana"
    networks:
      - obiente-network

  # Docker Registry - Host images for Swarm deployments
  registry:
    image: registry:2
    environment:
      REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /var/lib/registry
      REGISTRY_HTTP_ADDR: 0.0.0.0:5000
      # Allow deletion of images (useful for cleanup)
      REGISTRY_STORAGE_DELETE_ENABLED: "true"
      # Enable basic auth for private repositories
      REGISTRY_AUTH: htpasswd
      REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
      REGISTRY_AUTH_HTPASSWD_REALM: "Obiente Cloud Registry"
    volumes:
      - registry_data:/var/lib/registry
      # Use bind mount for auth directory to allow easy setup from host
      - /var/lib/obiente/registry-auth:/auth:ro
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 5000 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 5000 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s
    deploy:
      replicas: 1
      placement:
        constraints:
          - node.role == manager
      resources:
        limits:
          cpus: "2"
          memory: 512M
      reservations:
        cpus: "0.5"
        memory: 512M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.registry.rule=Host(`registry.${DOMAIN:-localhost}`)"
        - "traefik.http.routers.registry.entrypoints=websecure"
        - "traefik.http.routers.registry.tls.certresolver=letsencrypt"
        - "traefik.http.routers.registry.service=registry"
        - "traefik.http.services.registry.loadbalancer.server.port=5000"
    networks:
      - obiente-network

  # ============================================================================
  # MICROSERVICES
  # ============================================================================
  # All functionality has been moved from the monolithic API to microservices
  # API Gateway (port 3001) is the primary entry point and routes to microservices
  # Note: In HA setup, services connect to pgpool/metrics-pgpool instead of direct postgres/timescaledb
  # Note: Redis cluster connection - services should use redis-1, redis-2, or redis-3, or a Redis proxy

  # API Gateway - Routes requests to all microservices
  api-gateway:
    image: ghcr.io/obiente/cloud-api-gateway:latest
    environment:
      PORT: 3001
      <<: [*common-auth]
    ports:
      - target: 3001
        published: 3001
        protocol: tcp
        mode: ingress
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
        failure_action: rollback
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.api-gateway.rule=Host(`api.${DOMAIN}`)"
        - "traefik.http.routers.api-gateway.entrypoints=web"
        - "traefik.http.routers.api-gateway.service=api-gateway"
        # Retry middleware for transient failures
        - "traefik.http.middlewares.api-gateway-retry.retry.attempts=3"
        - "traefik.http.middlewares.api-gateway-retry.retry.initialInterval=100ms"
        # Circuit breaker removed from API gateway - gateway should always be accessible
        - "traefik.http.routers.api-gateway.middlewares=api-gateway-retry"
        - "traefik.http.services.api-gateway.loadbalancer.server.port=3001"
        - "traefik.http.services.api-gateway.loadbalancer.passHostHeader=true"
        - "traefik.http.services.api-gateway.loadbalancer.healthcheck.path=/health"
        - "traefik.http.services.api-gateway.loadbalancer.healthcheck.interval=10s"
        - "traefik.http.services.api-gateway.loadbalancer.healthcheck.timeout=8s"
        - "traefik.http.services.api-gateway.loadbalancer.healthcheck.scheme=http"
        - "traefik.http.services.api-gateway.loadbalancer.healthcheck.followRedirects=true"
        - "traefik.http.routers.api-gateway-secure.rule=Host(`api.${DOMAIN}`)"
        - "traefik.http.routers.api-gateway-secure.entrypoints=websecure"
        - "traefik.http.routers.api-gateway-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.api-gateway-secure.service=api-gateway"
        - "traefik.http.routers.api-gateway-secure.middlewares=api-gateway-retry"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3001 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3001 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Auth Service - Microservice for authentication
  auth-service:
    image: ghcr.io/obiente/cloud-auth-service:latest
    environment:
      PORT: 3002
      <<: [*common-database, *common-metrics-db, *common-auth, *common-dashboard]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.auth-service.rule=Host(`auth-service.${DOMAIN}`)"
        - "traefik.http.routers.auth-service.entrypoints=web"
        - "traefik.http.services.auth-service.loadbalancer.server.port=3002"
        - "traefik.http.routers.auth-service-secure.rule=Host(`auth-service.${DOMAIN}`)"
        - "traefik.http.routers.auth-service-secure.entrypoints=websecure"
        - "traefik.http.routers.auth-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.auth-service-secure.service=auth-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3002 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3002 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Organizations Service - Microservice for organization management
  organizations-service:
    image: ghcr.io/obiente/cloud-organizations-service:latest
    environment:
      PORT: 3003
      <<: [*common-database, *common-metrics-db, *common-auth, *common-smtp, *common-dashboard, *common-notifications]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.organizations-service.rule=Host(`organizations-service.${DOMAIN}`)"
        - "traefik.http.routers.organizations-service.entrypoints=web"
        - "traefik.http.services.organizations-service.loadbalancer.server.port=3003"
        - "traefik.http.routers.organizations-service-secure.rule=Host(`organizations-service.${DOMAIN}`)"
        - "traefik.http.routers.organizations-service-secure.entrypoints=websecure"
        - "traefik.http.routers.organizations-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.organizations-service-secure.service=organizations-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3003 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3003 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Billing Service - Microservice for billing and Stripe integration
  billing-service:
    image: ghcr.io/obiente/cloud-billing-service:latest
    environment:
      PORT: 3004
      <<: [*common-database, *common-metrics-db, *common-auth, *common-stripe, *common-dashboard, *common-notifications]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.billing-service.rule=Host(`billing-service.${DOMAIN}`)"
        - "traefik.http.routers.billing-service.entrypoints=web"
        - "traefik.http.services.billing-service.loadbalancer.server.port=3004"
        - "traefik.http.routers.billing-service-secure.rule=Host(`billing-service.${DOMAIN}`)"
        - "traefik.http.routers.billing-service-secure.entrypoints=websecure"
        - "traefik.http.routers.billing-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.billing-service-secure.service=billing-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3004 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3004 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Deployments Service - Microservice for deployment management
  deployments-service:
    image: ghcr.io/obiente/cloud-deployments-service:latest
    environment:
      PORT: 3005
      <<: [*common-database, *common-metrics-db, *common-auth, *common-redis, *common-orchestrator, *common-dns-delegation, *common-notifications]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
      REDIS_URL: ${REDIS_URL:-redis://redis-1:6379}
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /var/lib/obiente:/var/lib/obiente
      - /var/obiente/tmp/obiente-volumes:/tmp/obiente-volumes
      - /var/obiente/tmp/obiente-deployments:/tmp/obiente-deployments
    deploy:
      mode: global  # One instance per node for local Docker access
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
        failure_action: rollback
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "2"
          memory: 2G
        reservations:
          cpus: "1"
          memory: 512M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.deployments-service.rule=Host(`deployments-service.${DOMAIN}`)"
        - "traefik.http.routers.deployments-service.entrypoints=web"
        - "traefik.http.services.deployments-service.loadbalancer.server.port=3005"
        - "traefik.http.routers.deployments-service-secure.rule=Host(`deployments-service.${DOMAIN}`)"
        - "traefik.http.routers.deployments-service-secure.entrypoints=websecure"
        - "traefik.http.routers.deployments-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.deployments-service-secure.service=deployments-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3005 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3005 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Game Servers Service - Microservice for game server management
  gameservers-service:
    image: ghcr.io/obiente/cloud-gameservers-service:latest
    environment:
      PORT: 3006
      <<: [*common-database, *common-metrics-db, *common-auth, *common-notifications, *common-redis]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /var/lib/obiente/volumes:/var/lib/obiente/volumes
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.gameservers-service.rule=Host(`gameservers-service.${DOMAIN}`)"
        - "traefik.http.routers.gameservers-service.entrypoints=web"
        - "traefik.http.services.gameservers-service.loadbalancer.server.port=3006"
        - "traefik.http.routers.gameservers-service-secure.rule=Host(`gameservers-service.${DOMAIN}`)"
        - "traefik.http.routers.gameservers-service-secure.entrypoints=websecure"
        - "traefik.http.routers.gameservers-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.gameservers-service-secure.service=gameservers-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3006 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3006 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Orchestrator Service - Microservice for container orchestration and metrics
  orchestrator-service:
    image: ghcr.io/obiente/cloud-orchestrator-service:latest
    environment:
      PORT: 3007
      <<: [*common-database, *common-metrics-db, *common-redis, *common-orchestrator, *common-vps, *common-dns-delegation, *common-notifications]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
      REDIS_URL: ${REDIS_URL:-redis://redis-1:6379}
      ORCHESTRATOR_SYNC_INTERVAL: ${ORCHESTRATOR_SYNC_INTERVAL:-30s}
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "2"
          memory: 512M
        reservations:
          cpus: "1"
          memory: 512M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.orchestrator-service.rule=Host(`orchestrator-service.${DOMAIN}`)"
        - "traefik.http.routers.orchestrator-service.entrypoints=web"
        - "traefik.http.services.orchestrator-service.loadbalancer.server.port=3007"
        - "traefik.http.routers.orchestrator-service-secure.rule=Host(`orchestrator-service.${DOMAIN}`)"
        - "traefik.http.routers.orchestrator-service-secure.entrypoints=websecure"
        - "traefik.http.routers.orchestrator-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.orchestrator-service-secure.service=orchestrator-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3007 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3007 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # VPS Service - Microservice for VPS management
  vps-service:
    image: ghcr.io/obiente/cloud-vps-service:latest
    environment:
      PORT: 3008
      <<: [*common-database, *common-metrics-db, *common-auth, *common-orchestrator, *common-vps, *common-notifications]
      DB_HOST: pgpool
    secrets:
      - ssh_proxy_host_key
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.vps-service.rule=Host(`vps-service.${DOMAIN}`)"
        - "traefik.http.routers.vps-service.entrypoints=web"
        - "traefik.http.services.vps-service.loadbalancer.server.port=3008"
        - "traefik.http.routers.vps-service-secure.rule=Host(`vps-service.${DOMAIN}`)"
        - "traefik.http.routers.vps-service-secure.entrypoints=websecure"
        - "traefik.http.routers.vps-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.vps-service-secure.service=vps-service"
        # SSH TCP routing through Traefik (SSH proxy runs in vps-service)
        - "traefik.tcp.routers.ssh.rule=HostSNI(`*`)"
        - "traefik.tcp.routers.ssh.entrypoints=ssh"
        - "traefik.tcp.routers.ssh.service=ssh"
        - "traefik.tcp.services.ssh.loadbalancer.server.port=${SSH_PROXY_PORT:-2222}"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3008 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3008 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Support Service - Microservice for support ticket management
  support-service:
    image: ghcr.io/obiente/cloud-support-service:latest
    environment:
      PORT: 3009
      <<: [*common-database, *common-metrics-db, *common-auth]
      DB_HOST: pgpool
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.support-service.rule=Host(`support-service.${DOMAIN}`)"
        - "traefik.http.routers.support-service.entrypoints=web"
        - "traefik.http.services.support-service.loadbalancer.server.port=3009"
        - "traefik.http.routers.support-service-secure.rule=Host(`support-service.${DOMAIN}`)"
        - "traefik.http.routers.support-service-secure.entrypoints=websecure"
        - "traefik.http.routers.support-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.support-service-secure.service=support-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3009 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3009 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Audit Service - Microservice for audit log management
  audit-service:
    image: ghcr.io/obiente/cloud-audit-service:latest
    environment:
      PORT: 3010
      <<: [*common-database, *common-metrics-db, *common-auth]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.audit-service.rule=Host(`audit-service.${DOMAIN}`)"
        - "traefik.http.routers.audit-service.entrypoints=web"
        - "traefik.http.services.audit-service.loadbalancer.server.port=3010"
        - "traefik.http.routers.audit-service-secure.rule=Host(`audit-service.${DOMAIN}`)"
        - "traefik.http.routers.audit-service-secure.entrypoints=websecure"
        - "traefik.http.routers.audit-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.audit-service-secure.service=audit-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3010 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3010 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  notifications-service:
    image: ghcr.io/obiente/cloud-notifications-service:latest
    environment:
      PORT: 3012
      <<: [*common-database, *common-metrics-db, *common-auth, *common-smtp, *common-dashboard, *common-notifications]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
    deploy:
      mode: replicated
      replicas: 3
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.notifications-service.rule=Host(`notifications-service.${DOMAIN}`)"
        - "traefik.http.routers.notifications-service.entrypoints=web"
        - "traefik.http.services.notifications-service.loadbalancer.server.port=3012"
        - "traefik.http.routers.notifications-service-secure.rule=Host(`notifications-service.${DOMAIN}`)"
        - "traefik.http.routers.notifications-service-secure.entrypoints=websecure"
        - "traefik.http.routers.notifications-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.notifications-service-secure.service=notifications-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3012 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3012 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # Superadmin Service - Microservice for superadmin operations
  superadmin-service:
    image: ghcr.io/obiente/cloud-superadmin-service:latest
    environment:
      PORT: 3011
      <<: [*common-database, *common-metrics-db, *common-auth, *common-stripe, *common-orchestrator, *common-vps, *common-notifications]
      DB_HOST: pgpool
      METRICS_DB_HOST: metrics-pgpool
    deploy:
      mode: replicated
      replicas: 2
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.superadmin-service.rule=Host(`superadmin-service.${DOMAIN}`)"
        - "traefik.http.routers.superadmin-service.entrypoints=web"
        - "traefik.http.services.superadmin-service.loadbalancer.server.port=3011"
        - "traefik.http.routers.superadmin-service-secure.rule=Host(`superadmin-service.${DOMAIN}`)"
        - "traefik.http.routers.superadmin-service-secure.entrypoints=websecure"
        - "traefik.http.routers.superadmin-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.superadmin-service-secure.service=superadmin-service"
    healthcheck:
      test: ["CMD-SHELL", "sh -c 'if command -v nc >/dev/null 2>&1; then nc -z localhost 3011 || exit 1; else (apk add --no-cache netcat-openbsd >/dev/null 2>&1 || apt-get update -qq && apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || yum install -y -q nc >/dev/null 2>&1) && nc -z localhost 3011 || exit 1; fi'"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # DNS Service - Microservice for DNS resolution
  # Set ENABLE_DNS=false to disable this service when using DNS delegation
  dns-service:
    image: ghcr.io/obiente/cloud-dns-service:latest
    environment:
      <<: [*common-database, *common-redis, *common-orchestrator, *common-dns, *common-dns-delegation]
      DB_HOST: pgpool
      HTTP_PORT: ${DNS_HTTP_PORT:-8053}
      REDIS_URL: ${REDIS_URL:-redis://redis-1:6379}
    ports:
      # Use host mode publishing to expose port 53 directly on the host
      # DNS only runs on nodes labeled with dns.enabled=true
      - target: 53
        published: 53
        protocol: udp
        mode: host
      - target: 53
        published: 53
        protocol: tcp
        mode: host
    cap_add:
      - NET_BIND_SERVICE
    deploy:
      mode: replicated
      replicas: ${DNS_REPLICAS:-1}
      placement:
        constraints:
          - node.labels.dns.enabled == true
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
      labels:
        - "cloud.obiente.service=dns-service"
        - "cloud.obiente.traefik=true"
        - "traefik.enable=true"
        - "traefik.http.routers.dns-service.rule=Host(`dns-service.${DOMAIN}`)"
        - "traefik.http.routers.dns-service.entrypoints=web"
        - "traefik.http.services.dns-service.loadbalancer.server.port=8053"
        - "traefik.http.routers.dns-service-secure.rule=Host(`dns-service.${DOMAIN}`)"
        - "traefik.http.routers.dns-service-secure.entrypoints=websecure"
        - "traefik.http.routers.dns-service-secure.tls.certresolver=letsencrypt"
        - "traefik.http.routers.dns-service-secure.service=dns-service"
        - "traefik.docker.network=obiente_obiente-network"  # Use overlay network IP, not ingress network
    healthcheck:
      # Check HTTP port (8053) instead of DNS port (53) since HTTP server always runs
      # even when ENABLE_DNS=false, and this is what Traefik routes to
      test: ["CMD-SHELL", "curl -sf http://localhost:${HTTP_PORT:-8053}/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    dns_search: []
    networks:
      - obiente-network

  # VPS Gateway - gRPC gateway for VPS operations (optional)
  vps-gateway:
    image: ghcr.io/obiente/cloud-vps-gateway:latest
    ports:
      - target: 1537
        published: 1537
        protocol: tcp
        mode: ingress
      - target: 9091
        published: 9091
        protocol: tcp
        mode: ingress
    deploy:
      mode: replicated
      replicas: 1
      placement:
        constraints:
          - node.labels.vps-gateway.enabled == true
      update_config:
        parallelism: 1
        delay: 10s
        order: start-first
      restart_policy:
        condition: on-failure
        delay: 10s
        max_attempts: 10
        window: 60s
      resources:
        limits:
          cpus: "1"
          memory: 512M
        reservations:
          cpus: "0.5"
          memory: 256M
    dns_search: []
    networks:
      - obiente-network

  # Note: vps-gateway service is deployed separately using docker-compose.vps-gateway.yml
  # It should be deployed on the gateway VM/container with network_mode: host

# ============================================================================
# VOLUMES
# ============================================================================

volumes:
  # etcd
  etcd_data:
    driver: local

  # PostgreSQL Patroni
  patroni_1_data:
    driver: local
  patroni_2_data:
    driver: local
  patroni_3_data:
    driver: local

  # TimescaleDB Patroni
  metrics_patroni_1_data:
    driver: local
  metrics_patroni_2_data:
    driver: local
  metrics_patroni_3_data:
    driver: local

  # Redis Cluster
  redis_1_data:
    driver: local
  redis_2_data:
    driver: local
  redis_3_data:
    driver: local

  # Traefik
  traefik_letsencrypt:
    driver: local

  # Monitoring
  prometheus_data:
    driver: local
  grafana_data:
    driver: local

  # Docker Registry
  registry_data:
    driver: local

  # VPS Gateway

# ============================================================================
# NETWORKS
# ============================================================================

networks:
  obiente-network:
    driver: overlay
    attachable: true
    driver_opts:
      encrypted: "true"
    ipam:
      config:
        - subnet: 10.15.3.0/24
    # Docker Swarm will automatically create this network when the stack is deployed
    # Network name will be: {stack-name}_obiente-network (e.g., obiente_obiente-network)

# ============================================================================
# SECRETS
# ============================================================================

secrets:
  ssh_proxy_host_key:
    external: true
    # This secret must be created before deploying the stack:
    # 1. Generate a key: ssh-keygen -t rsa -b 2048 -f ssh_proxy_host_key -N ""
    # 2. Create the secret: docker secret create ssh_proxy_host_key ssh_proxy_host_key
    # 3. Remove the local file: rm ssh_proxy_host_key ssh_proxy_host_key.pub
    #
    # This ensures all vps-service replicas use the same SSH host key,
    # preventing "REMOTE HOST IDENTIFICATION HAS CHANGED" warnings for clients.