Varnish + nginx for tiler cache (#747)

* Add varnish config

* Update varnish config and remove nginx cache

* Update traefik for varnish

* Update congif forvarnish y clean the tiles in a bash

* Update varnish purge config

* Fix varnish config

* Fix tiler viewer

* Remove unused env vars for tiler-cache

* Update docker config

* Remove logic to remove tiler from s3 - tegola

* Rename varnish folder

* Fix VARNISH_TILE_URL_PREFIX to pass ohm, ohm_admin and ohm_other_boundaries

* Update config to deploy

Author: Rub21

.github/workflows/chartpress.yaml

@@ -4,7 +4,7 @@ on:
     branches:
       - 'main'
       - 'staging'
-      - 'req_fresh_tiles'
+      - 'varnish_tiler'
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -71,7 +71,7 @@ jobs:
           OHM_SLACK_WEBHOOK_URL: ${{ secrets.OHM_SLACK_WEBHOOK_URL }}
       ################ Staging secrets ################ 
       - name: Staging - substitute secrets
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/req_fresh_tiles'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/varnish_tiler'
         uses: bluwy/substitute-string-action@v1
         with:
           _input-file: 'values.staging.template.yaml'
@@ -189,14 +189,14 @@ jobs:
           PRODUCTION_OPENSTREETMAP_AUTH_SECRET: ${{ secrets.PRODUCTION_OPENSTREETMAP_AUTH_SECRET }}
 
       - name: AWS Credentials
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/req_fresh_tiles'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/varnish_tiler'
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
       - name: Setup Kubectl and Helm Dependencies
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/req_fresh_tiles'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/varnish_tiler'
         run: |
           sudo pip install awscli --ignore-installed six
           sudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl
@@ -210,22 +210,22 @@ jobs:
           helm version
 
       - name: Update kube-config staging
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/req_fresh_tiles'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/varnish_tiler'
         run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging
       - name: Update kube-config prod
         if: github.ref == 'refs/heads/main'
         run: aws eks --region us-east-1 update-kubeconfig --name osmseed-production-v2
       - name: Add Helm repository
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/req_fresh_tiles'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/varnish_tiler'
         run: |
           helm repo add osm-seed https://osm-seed.github.io/osm-seed-chart/
           helm repo update
       - name: Install helm dependencies for
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/req_fresh_tiles'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/varnish_tiler'
         run: cd ohm && helm dep up
       # Staging
       - name: Staging - helm deploy
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/req_fresh_tiles'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/varnish_tiler'
         run: helm upgrade --install staging --wait ohm/ -f values.staging.yaml -f ohm/values.yaml
       # Production
       - name: Production - helm deploy

compose/martin.yml

@@ -1,7 +1,25 @@
-# Martin tile server
-# Cache invalidation: nginx TTLs per zoom + manual purge via ?purge=1 query param.
-# No external purge service needed.
+# Martin tile server + Varnish cache
+# Caching: Varnish only. Nginx in Martin does routing/gzip/tilejson/static.
+# Invalidation: BAN requests from tiler-cache (replaces TTL-based invalidation).
 services:
+  varnish:
+    image: varnish:7.5
+    container_name: varnish
+    ports:
+      - "6081:6081"
+    volumes:
+      - ../images/tiler-varnish/default.vcl:/etc/varnish/default.vcl:ro
+    command: >
+      varnishd -F
+      -a :6081
+      -f /etc/varnish/default.vcl
+      -s dynamic=malloc,${VARNISH_DYNAMIC_SIZE:-2G}
+      -s static=malloc,${VARNISH_STATIC_SIZE:-1G}
+    networks:
+      - ohm_network
+    depends_on:
+      - martin
+
   martin:
     container_name: martin
     image: rub21/tiler-server-martin:v1
@@ -10,21 +28,51 @@ services:
       dockerfile: Dockerfile
     volumes:
       - ../images/tiler-server-martin:/app
-      - martin_nginx_cache:/var/cache/nginx
     ports:
       - "3020:80"
+    expose:
+      - "3001"
     env_file:
       - ../envs/.env.tiler
     environment:
       - OHM_DOMAIN=${OHM_DOMAIN:-openhistoricalmap.org}
-    # restart: always
     networks:
       - ohm_network
 
-volumes:
-  martin_nginx_cache:
-    driver: local
-    name: martin_nginx_cache
+  tiler-cache:
+    container_name: tiler-cache
+    image: ohm/tiler-cache:latest
+    build:
+      context: ../images/tiler-cache
+      dockerfile: Dockerfile
+    volumes:
+      - ../images/tiler-cache:/app
+    ports:
+      - "8000:8000"
+    command:
+      - /bin/sh
+      - -c
+      - |
+        set -x
+        python sqs_processor.py &
+        python main.py
+    env_file:
+      - ../envs/.env.tiler
+    environment:
+      - PORT=8000
+      - TILE_CACHE_BACKEND=varnish
+      - VARNISH_URL=http://varnish:6081
+    restart: always
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health').read()"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    networks:
+      - ohm_network
+    depends_on:
+      - varnish
 
 networks:
   ohm_network:

hetzner/start_all.sh

@@ -20,7 +20,7 @@ echo "########################## ENVIRONMENT -> $ENVIRONMENT ###################
 source "$SCRIPT_DIR/.env.traefik"
 echo "########################## OHM_DOMAIN -> $OHM_DOMAIN ##########################"
 
-# ###################### Tiler ######################
+###################### Tiler ######################
 ./hetzner/deploy.sh start tiler $ENVIRONMENT -y
 
 # ###################### Osmcha ######################

hetzner/tiler/env.production.example

@@ -47,25 +47,13 @@ TILER_CACHE_AWS_SECRET_ACCESS_KEY=hetzner_credentials
 TILER_CACHE_AWS_ENDPOINT=https://hel1.your-objectstorage.com
 
 # #######################################
-# tiler cache
+# tiler cache (Varnish BAN invalidator)
 # #######################################
 AWS_REGION_NAME=us-east-1
 AWS_ACCESS_KEY_ID=xxxx
 AWS_SECRET_ACCESS_KEY=yyyyy
 SQS_QUEUE_URL=https://sqs.us-east-1.amazonaws.com/1234567890/tiler-imposm3
 ENVIRONMENT=production
-DOCKER_IMAGE=none
-NODEGROUP_TYPE=web_large
-MAX_ACTIVE_JOBS=10
-DELETE_OLD_JOBS_AGE=3600
-EXECUTE_PURGE=true
-EXECUTE_SEED=false
-PURGE_MIN_ZOOM=3
-PURGE_MAX_ZOOM=10
-SEED_MIN_ZOOM=0
-SEED_MAX_ZOOM=8
-SEED_CONCURRENCY=64
-PURGE_CONCURRENCY=64
 ZOOM_LEVELS_TO_DELETE=8,9,10,11,12,13,14,15,16,17,18,19,20
-S3_BUCKET_CACHE_TILER=tiler-cache-bucket
-S3_BUCKET_PATH_FILES=mnt/data/osm
+ENABLE_DELAYED_CLEANUP=true
+VARNISH_URL=http://varnish:6081

hetzner/tiler/tiler.base.yml

@@ -1,18 +1,16 @@
 services:
-  tiler_server:
-    container_name: tiler_server
-    image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.3296.h0e45d176
-    # image: tiler-server:staging
-    # build:
-    #   context: ../../images/tiler-server
-    #   dockerfile: Dockerfile
-    # ports:
-    #   - "9090:9090"
+  tiler_server_martin:
+    container_name: tiler_server_martin
+    image:  ghcr.io/openhistoricalmap/tiler-server-martin:0.0.1-0.dev.git.3352.h1c81647b
+    restart: always
+    environment:
+      - OHM_DOMAIN=${OHM_DOMAIN:-openhistoricalmap.org}
     env_file:
       - .env.tiler
-    restart: always
-    # volumes:
-    #   - ../../images/tiler-server/start.sh:/app/start.sh:ro
+    ports:
+      - "3030:80"
+    volumes:
+      - tiler_martin_nginx_cache:/var/cache/nginx
     networks:
       - ohm_network
 

hetzner/tiler/tiler.production.yml

@@ -37,19 +37,9 @@ services:
     networks:
       - ohm_network
 
-  # Tiler server
-  tiler_server:
-    container_name: tiler_server
-    image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.3292.h0f5e7d60
-    # ports:
-    #   - "9090:9090"
-    restart: always
-    networks:
-      - ohm_network
-
   tiler_sqs_cleaner:
     container_name: tiler_sqs_cleaner
-    image: ghcr.io/openhistoricalmap/tiler-cache:0.0.1-0.dev.git.3290.h63549fc1
+    image: ghcr.io/openhistoricalmap/tiler-cache:0.0.1-0.dev.git.3357.h3c23a908
     environment:
       - PORT=8000
     env_file:
@@ -70,55 +60,6 @@ services:
     networks:
       - ohm_network
 
-  tiler_s3_cleaner:
-    container_name: tiler_s3_cleaner
-    image: ghcr.io/openhistoricalmap/tiler-cache:0.0.1-0.dev.git.3290.h63549fc1
-    # image: tiler-cache:latest
-    # build:
-    #   context: ../../images/tiler-cache
-    #   dockerfile: Dockerfile
-    command:
-      - /bin/sh
-      - -c
-      - |
-        tiler-cache-cleaner clean_by_prefix --prefix-path-file mnt/data/ohm
-        tiler-cache-cleaner clean_by_prefix --prefix-path-file mnt/data/ohm_admin
-        tiler-cache-cleaner clean_by_prefix --prefix-path-file mnt/data/ohm_other_boundaries
-    env_file:
-      - .env.tiler
-    networks:
-      - ohm_network
-
-  tile_global_seeding:
-    container_name: tiler_global_seeding
-    image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.3296.h0e45d176
-    env_file:
-      - .env.tiler
-    volumes:
-      - ./seed.sh:/opt/seed.sh
-    entrypoint:
-      - /bin/bash
-      - "-c"
-      - |
-        /opt/seed.sh global
-    networks:
-      - ohm_network
-
-  tile_coverage_seeding:
-    container_name: tiler_coverage_seeding
-    image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.3296.h0e45d176
-    volumes:
-      - ./seed.sh:/opt/seed.sh
-    entrypoint:
-      - /bin/bash
-      - "-c"
-      - |
-        /opt/seed.sh coverage
-    env_file:
-      - .env.tiler
-    networks:
-      - ohm_network
-
   tiler_monitor:
     container_name: tiler_monitor
     image: ghcr.io/openhistoricalmap/tiler-monitor:0.0.1-0.dev.git.3348.h4fbb83d8
@@ -138,7 +79,7 @@ services:
 
   tiler_server_martin:
     container_name: tiler_server_martin
-    image: ghcr.io/openhistoricalmap/tiler-server-martin:0.0.1-0.dev.git.3357.he67b0015
+    image:  ghcr.io/openhistoricalmap/tiler-server-martin:0.0.1-0.dev.git.3352.h1c81647b
     restart: always
     environment:
       - OHM_DOMAIN=${OHM_DOMAIN:-openhistoricalmap.org}

hetzner/tiler/tiler.staging.yml

@@ -1,42 +1,5 @@
 services:
-  tiler_db:
-    container_name: tiler_db
-    image: ghcr.io/openhistoricalmap/tiler-db:0.0.1-0.dev.git.2166.hc55c4cd
-    volumes:
-      - tiler_pgdata:/var/lib/postgresql/data
-      - ./config/postgresql.staging.conf:/etc/postgresql/postgresql.conf
-    environment:
-      - PGDATA=/var/lib/postgresql/data
-      - POSTGRES_CONFIG_FILE=/etc/postgresql/postgresql.conf
-    command:
-      - postgres
-      - "-c"
-      - "config_file=/etc/postgresql/postgresql.conf"
-    ports:
-      - "5432:5432"
-    env_file:
-      - .env.tiler
-    networks:
-      - ohm_network
-  tiler_imposm:
-    container_name: tiler_imposm
-    # image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.3208.h6ef73bb
-    image: tiler-imposm:staging
-    build: 
-      context: ../../images/tiler-imposm
-      dockerfile: Dockerfile
-    volumes:
-      - tiler_imposm_data:/mnt/data
-    command:
-      - sh
-      - -c
-      - |
-        ./start.sh
-    env_file:
-      - .env.tiler
-    restart: always
-    networks:
-      - ohm_network
+
 volumes:
   tiler_pgdata:
     driver: local

hetzner/traefik/traefik.template.yml

@@ -84,7 +84,7 @@ http:
       rule: Host(`vtiles.{{OHM_DOMAIN}}`)
       entryPoints:
         - port-web
-      service: tiler_server_martin
+      service: varnish_tiles
       middlewares:
         - secure-headers-allow-iframe
         - replace-osm-tiles-to-ohm
@@ -130,14 +130,6 @@ http:
       middlewares:
         - secure-headers
 
-    martin-router:
-      rule: Host(`martin.{{OHM_DOMAIN}}`)
-      entryPoints:
-        - port-web
-      service: tiler_server_martin
-      middlewares:
-        - secure-headers
-
     node-exporter-router:
       rule: Host(`node-exporter.{{OHM_DOMAIN}}`)
       entryPoints:
@@ -193,10 +185,10 @@ http:
         servers:
           - url: http://taginfo_web:4567
 
-    tiler_server_martin:
+    varnish_tiles:
       loadBalancer:
         servers:
-          - url: http://tiler_server_martin:80
+          - url: http://varnish:6081
 
     node_exporter:
       loadBalancer:

images/tiler-cache/Dockerfile

@@ -1,11 +1,5 @@
 FROM python:3.10
 
-# Install kubectl
-ARG KUBECTL_VERSION=v1.30.1
-RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
-    && chmod +x kubectl \
-    && mv kubectl /usr/local/bin/
-RUN kubectl version --client
 WORKDIR /app
 RUN echo "installing requirements"
 COPY ./requirements.txt .