Merge pull request #739 from OpenHistoricalMap/staging

Tiler monitoring pipeline, server migration & operational improvements

Author: Rub21

.github/workflows/chartpress.yaml

@@ -4,7 +4,7 @@ on:
     branches:
       - 'main'
       - 'staging'
-      - 'k8s_deploy'
+      - 'tiler_monitoring'
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -71,7 +71,7 @@ jobs:
           OHM_SLACK_WEBHOOK_URL: ${{ secrets.OHM_SLACK_WEBHOOK_URL }}
       ################ Staging secrets ################ 
       - name: Staging - substitute secrets
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
         uses: bluwy/substitute-string-action@v1
         with:
           _input-file: 'values.staging.template.yaml'
@@ -189,46 +189,46 @@ jobs:
           PRODUCTION_OPENSTREETMAP_AUTH_SECRET: ${{ secrets.PRODUCTION_OPENSTREETMAP_AUTH_SECRET }}
 
       - name: AWS Credentials
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
       - name: Setup Kubectl and Helm Dependencies
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
         run: |
           sudo pip install awscli --ignore-installed six
-          sudo curl -L -o /usr/bin/kubectl https://dl.k8s.io/release/v1.33.0/bin/linux/amd64/kubectl
+          sudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl
           sudo chmod +x /usr/bin/kubectl
-          sudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.21.2/2021-07-05/bin/linux/amd64/aws-iam-authenticator
+          sudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator
           sudo chmod +x /usr/bin/aws-iam-authenticator
-          curl -L https://get.helm.sh/helm-v3.17.3-linux-amd64.tar.gz -o helm.tar.gz
+          curl -L https://get.helm.sh/helm-v3.14.4-linux-amd64.tar.gz -o helm.tar.gz
           tar -xvzf helm.tar.gz
           sudo mv linux-amd64/helm /usr/local/bin/
           sudo chmod +x /usr/local/bin/helm
           helm version
 
       - name: Update kube-config staging
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
         run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging
       - name: Update kube-config prod
         if: github.ref == 'refs/heads/main'
         run: aws eks --region us-east-1 update-kubeconfig --name osmseed-production-v2
       - name: Add Helm repository
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
         run: |
           helm repo add osm-seed https://osm-seed.github.io/osm-seed-chart/
           helm repo update
       - name: Install helm dependencies for
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/k8s_deploy'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
         run: cd ohm && helm dep up
       # Staging
       - name: Staging - helm deploy
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/k8s_deploy'
-        run: helm upgrade --install staging --wait --timeout 10m ohm/ -f values.staging.yaml -f ohm/values.yaml
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
+        run: helm upgrade --install staging --wait ohm/ -f values.staging.yaml -f ohm/values.yaml
       # Production
       - name: Production - helm deploy
         if: github.ref == 'refs/heads/main'
-        run: helm upgrade --install production --wait --timeout 10m ohm/ -f values.production.yaml -f ohm/values.yaml
+        run: helm upgrade --install production --wait ohm/ -f values.production.yaml -f ohm/values.yaml
 

.gitignore

@@ -33,8 +33,8 @@ hetzner/*/.envs.*.production
 hetzner/traefik/cloudflare-ips.txt
 hetzner/traefik/traefik.yml
 .vscode/
+.claude/
 imposm3.json
-
 cachedir_reimport/
 config_reimport.json
 imposm3_reimport.json

compose/tiler.yml

@@ -71,21 +71,24 @@ services:
   #     - ohm_network
 
 
-  # tiler-monitor:
-  #   image: rub21/tiler-monitor:v1
-  #   build:
-  #     context: ../images/tiler-monitor
-  #     dockerfile: Dockerfile
-  #   volumes:
-  #     - /var/run/docker.sock:/var/run/docker.sock
-  #     - ../images/tiler-monitor:/app
-  #     - ../hetzner:/app/hetzner
-  #   environment:
-  #     - DOCKER_CONFIG_ENVIRONMENT=staging
-  #   env_file:
-  #     - ../envs/.env.tiler
-  #   stdin_open: true
-  #   tty: true
+  tiler-monitor:
+    image: rub21/tiler-monitor:v2
+    build:
+      context: ../images/tiler-monitor
+      dockerfile: Dockerfile
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ../hetzner:/app/hetzner
+      - tiler_monitor_data:/data
+    ports:
+      - "8001:8001"
+    environment:
+      - TILER_MONITORING_DOCKER_CONFIG_ENVIRONMENT=staging
+    env_file:
+      - ../envs/.env.tiler
+    restart: always
+    networks:
+      - ohm_network
 
 networks:
   ohm_network:
@@ -99,3 +102,7 @@ volumes:
   tiler_imposm_data:
     driver: local
     name: tiler_imposm
+
+  tiler_monitor_data:
+    driver: local
+    name: tiler_monitor

hetzner/nominatim/nominatim.base.yml

@@ -5,7 +5,7 @@ services:
     volumes:
       - nominatim_data:/var/lib/postgresql/16/main
     environment:
-      PBF_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250716_0101.osm.pbf
+      PBF_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260408_0307.osm.pbf
       REPLICATION_URL: http://planet.openhistoricalmap.org.s3.amazonaws.com/replication/minute
       REPLICATION_UPDATE_INTERVAL: 60
       REPLICATION_RECHECK_INTERVAL: 30

hetzner/osmcha/osmcha.base.yml

@@ -91,6 +91,7 @@ services:
     volumes:
       - staticfiles:/srv/www/static/django:ro
       - ${PWD}/hetzner/osmcha/script/update.sh:/app/update.sh
+      - ${PWD}/hetzner/osmcha/script/backfill_changesets.py:/app/osmchadjango/changeset/management/commands/backfill_changesets.py
     command: sh /app/update.sh
 
 volumes:

hetzner/osmcha/osmcha.production.yml

@@ -4,10 +4,10 @@ services:
     container_name: osmcha_ohmx_adiff
     environment:
       - API_URL=https://api.${OHM_DOMAIN}
-      - PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260106_0350.osm.pbf
+      - PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260408_0307.osm.pbf
       - MINUTE_REPLICATION_URL=https://planet.openhistoricalmap.org/?prefix=replication/minute/
       # Add OSMX_INITIAL_SEQNUM to start from a specific sequence number
-      # - OSMX_INITIAL_SEQNUM=1881020
+      # - OSMX_INITIAL_SEQNUM=1900000
       - AWS_S3_BUCKET=planet.openhistoricalmap.org
     env_file:
       - ./.env.osmcha
@@ -22,5 +22,5 @@ services:
       - ${PWD}/images/ohmx-adiff-builder/update.sh:/app/update.sh
     networks:
       - ohm_network
-    cpus: '8.0'
-    mem_limit: 20G
+    # cpus: '8.0'
+    # mem_limit: 20G

hetzner/osmcha/script/backfill_changesets.py

@@ -0,0 +1,77 @@
+"""Django management command to backfill changesets, customized from the original one."""
+
+from datetime import date, datetime, timedelta
+from django.core.management.base import BaseCommand
+
+from osmchadjango.changeset.models import Changeset
+from osmchadjango.changeset.tasks import create_changeset
+
+class Command(BaseCommand):
+    help = """Backfill missing changesets by ID range or date range.
+        Use --start_id for specific ID, --step=1/-1 for direction."""
+
+    def add_arguments(self, parser):
+        parser.add_argument("--start_date", type=str)
+        parser.add_argument("--end_date", type=str)
+        parser.add_argument("--start_id", type=int)
+        parser.add_argument("--limit", type=int, default=50)
+        parser.add_argument("--step", type=int, default=1)
+
+    def handle(self, *args, **options):
+        # Date defaults
+        try:
+            start_date = date.fromisoformat(options["start_date"])
+        except:
+            start_date = date.today() - timedelta(days=1)
+        
+        try:
+            end_date = date.fromisoformat(options["end_date"])
+        except:
+            end_date = datetime.now().date()
+        
+        # Priority: ID mode > date mode
+        if options["start_id"]:
+            start_id = options["start_id"]
+            # Get local context (recent IDs around start_id)
+            cl = list(Changeset.objects.filter(id__gte=start_id - 100).values_list("id", flat=True))
+            max_id = max(cl) if cl else start_id
+            min_id = min(cl) if cl else start_id + options["step"]
+            self.stdout.write(f"Backfilling from ID {start_id}, range {max_id} ↔ {min_id}")
+            
+            current_id = start_id
+            count = 0
+            while count < options["limit"]:
+                if current_id not in cl:
+                    try:
+                        create_changeset(current_id)
+                        self.stdout.write(f"✅ {current_id}")
+                    except Exception as e:
+                        self.stdout.write(f"✗ {current_id}: {e}")
+                current_id += options["step"]
+                count += 1
+        else:
+            # Original date-based logic + empty list FIX
+            cl_qs = Changeset.objects.filter(date__gte=start_date, date__lte=end_date).values_list("id", flat=True)
+            cl = list(cl_qs)
+            
+            if not cl:
+                self.stdout.write("No changesets in date range. Use --start_id=25")
+                return
+            
+            max_id = max(cl)
+            min_id = min(cl)
+            self.stdout.write(f"Found range: {min_id} ↔ {max_id}")
+            
+            current_id = max_id + 1
+            count = 0
+            while current_id < min_id and count < options["limit"]:
+                try:
+                    create_changeset(current_id)
+                    self.stdout.write(f"✅ {current_id}")
+                except Exception as e:
+                    self.stdout.write(f"✗ {current_id}: {e}")
+                current_id += 1
+                count += 1
+
+        self.stdout.write("✅ Complete")
+        

hetzner/overpass/overpass.base.yml

@@ -9,10 +9,10 @@ services:
     environment:
       OVERPASS_META: attic
       OVERPASS_MODE: init
-      OVERPASS_PLANET_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260216_0301.osm.pbf
+      OVERPASS_PLANET_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260408_0307.osm.pbf
       OVERPASS_DIFF_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute
       OVERPASS_RULES_LOAD: 10
-      OVERPASS_REPLICATION_SEQUENCE_NUMBER: 1796000
+      OVERPASS_REPLICATION_SEQUENCE_NUMBER: 1900000
       OVERPASS_ALLOW_DUPLICATE_QUERIES: yes
     restart: always
     ## Disable healthcheck during initialization phase to prevent premature restarts

hetzner/overpass/overpass.production.yml

@@ -1,4 +1,4 @@
 services:
   overpass:
-    mem_limit: 16g
-    cpus: "8.0"
+    # mem_limit: 16g
+    # cpus: "8.0"

hetzner/tiler/config/postgresql.production.conf

@@ -2,18 +2,18 @@
 # CONNECTIONS AND AUTHENTICATION
 #------------------------------------------------------------------------------
 listen_addresses = '*'                    # Allow connections from any network interface
-max_connections = 200                      # Increase if you expect more concurrent connections
+max_connections = 300                      # 4 Tegola replicas × 56 conn + refresh/imposm headroom
 superuser_reserved_connections = 5         # Reserve connections for superusers
 
 #------------------------------------------------------------------------------
 # RESOURCE USAGE
 #------------------------------------------------------------------------------
 
 # - Memory Configuration -
-shared_buffers = 10GB                      # ~25% of 40GB container limit
-work_mem = 256MB                           # Memory for each sort/hash operation; be cautious with many parallel queries
-maintenance_work_mem = 4GB                 # Larger memory for VACUUM / CREATE INDEX / ALTER
-effective_cache_size = 26GB                # ~65% of 40GB container limit
+shared_buffers = 16GB                      # ~13% of 125GB RAM
+work_mem = 256MB                           # Memory for each sort/hash operation; balanced for 300 max connections
+maintenance_work_mem = 8GB                 # Larger memory for VACUUM / CREATE INDEX / ALTER / REFRESH
+effective_cache_size = 80GB                # ~65% of 125GB RAM
 
 # - Disk Optimization for SSD (if using SSD) -
 random_page_cost = 1.0                     # Lower cost for random I/O on SSD
@@ -32,7 +32,7 @@ synchronous_commit = off                   # Improves write performance, risk of
 #------------------------------------------------------------------------------
 # AUTOVACUUM SETTINGS
 #------------------------------------------------------------------------------
-autovacuum_max_workers = 6                 # More parallel vacuum workers for busy systems
+autovacuum_max_workers = 8                 # More parallel vacuum workers for busy systems
 autovacuum_naptime = 30s                   # How often the autovacuum daemon checks for work
 autovacuum_vacuum_cost_limit = -1          # Let PostgreSQL adjust vacuum cost dynamically
 
@@ -42,26 +42,27 @@ autovacuum_vacuum_cost_limit = -1          # Let PostgreSQL adjust vacuum cost d
 effective_io_concurrency = 300             # For SSD; helps the planner estimate IO concurrency
 parallel_tuple_cost = 0.001                # Lower cost to encourage parallelization
 parallel_setup_cost = 100                  # Lower to encourage more parallel plans
-max_worker_processes = 25                  # Match 25 CPUs container limit
-max_parallel_workers_per_gather = 6        # Max workers that can help a single query
-max_parallel_workers = 25                  # Total number of parallel workers across all queries
+max_worker_processes = 36                  # Match 36 CPUs
+max_parallel_workers_per_gather = 8        # Max workers that can help a single query
+max_parallel_workers = 36                  # Total number of parallel workers across all queries
 
 #------------------------------------------------------------------------------
 # LOGGING
 #------------------------------------------------------------------------------
 logging_collector = off                   # Disable log collection
+log_min_messages = fatal                   # Only log FATAL and PANIC (crashes, OOM kills)
+log_min_error_statement = panic            # Do not log SQL statements even on errors
 log_statement = 'none'                     # Do not log any statements
 log_duration = off                         # Disable logging query duration
 log_min_duration_statement = -1            # Disable logging slow queries
-log_error_verbosity = terse                # Minimal error messages
+log_error_verbosity = default              # Show enough detail for debugging errors
 log_autovacuum_min_duration = -1           # Do not log autovacuum runs
-log_connections = on                      # Do not log new connections
-log_disconnections = on                   # Do not log disconnections
+log_connections = off                      # Do not log connections
+log_disconnections = off                   # Do not log disconnections
 log_lock_waits = off                       # Do not log lock waits
 log_temp_files = -1                        # Do not log temporary file creation
 log_checkpoints = off                      # Do not log checkpoints
 log_replication_commands = off             # Do not log replication-related commands
-log_directory = '/dev/null'                # Redirect logs to /dev/null (no storage)
 #------------------------------------------------------------------------------
 # CLIENT CONNECTION DEFAULTS
 #------------------------------------------------------------------------------