Improve imposm config management and migrate tiler to new server (#736)

* Upload imposm config to s3

* Update branch name to create docker image

* Read imposm config from s3

* Update config  to fetch imposm3.json

* Add env var to refresh views in parallel or sequencial

* Increase work_men and maint_men for update mviews in seqeunce

* Refresh all mviews in sequence

* Set local for SECONDS

* Update

* mview refresh

* Add max_parallel_workers_per_gather to refresh views

* Update config

* Update shared_buffers for tiler db

* Keep imposm logs from each restart

* Fix imposm liveness

* Remove last.state.txt from imposm dir

* Chekc object type to avoid flase positive

* Print warnings when a object is not imported in the db

* Reject naturla=coastline for landuse areas

* Refresh views in parallel - and post the status in s3

* Update config for new tiler server

* Migrate services to a new server

* Update reseource for porduction

* Update db connetion for tiler -  new server

* Add monitoring route - cloudflare

Author: Rub21

.github/workflows/chartpress.yaml

@@ -4,7 +4,7 @@ on:
     branches:
       - 'main'
       - 'staging'
-      - 'tiler_monitoring'
+      - 'imposm_config'
 jobs:
   build:
     runs-on: ubuntu-22.04
@@ -71,7 +71,7 @@ jobs:
           OHM_SLACK_WEBHOOK_URL: ${{ secrets.OHM_SLACK_WEBHOOK_URL }}
       ################ Staging secrets ################ 
       - name: Staging - substitute secrets
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/imposm_config'
         uses: bluwy/substitute-string-action@v1
         with:
           _input-file: 'values.staging.template.yaml'
@@ -189,14 +189,14 @@ jobs:
           PRODUCTION_OPENSTREETMAP_AUTH_SECRET: ${{ secrets.PRODUCTION_OPENSTREETMAP_AUTH_SECRET }}
 
       - name: AWS Credentials
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/imposm_config'
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
       - name: Setup Kubectl and Helm Dependencies
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/imposm_config'
         run: |
           sudo pip install awscli --ignore-installed six
           sudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl
@@ -210,22 +210,22 @@ jobs:
           helm version
 
       - name: Update kube-config staging
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/imposm_config'
         run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging
       - name: Update kube-config prod
         if: github.ref == 'refs/heads/main'
         run: aws eks --region us-east-1 update-kubeconfig --name osmseed-production-v2
       - name: Add Helm repository
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/imposm_config'
         run: |
           helm repo add osm-seed https://osm-seed.github.io/osm-seed-chart/
           helm repo update
       - name: Install helm dependencies for
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/tiler_monitoring'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/imposm_config'
         run: cd ohm && helm dep up
       # Staging
       - name: Staging - helm deploy
-        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/tiler_monitoring'
+        if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/imposm_config'
         run: helm upgrade --install staging --wait ohm/ -f values.staging.yaml -f ohm/values.yaml
       # Production
       - name: Production - helm deploy

hetzner/nominatim/nominatim.base.yml

@@ -5,7 +5,7 @@ services:
     volumes:
       - nominatim_data:/var/lib/postgresql/16/main
     environment:
-      PBF_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250716_0101.osm.pbf
+      PBF_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260408_0307.osm.pbf
       REPLICATION_URL: http://planet.openhistoricalmap.org.s3.amazonaws.com/replication/minute
       REPLICATION_UPDATE_INTERVAL: 60
       REPLICATION_RECHECK_INTERVAL: 30

hetzner/osmcha/osmcha.production.yml

@@ -4,10 +4,10 @@ services:
     container_name: osmcha_ohmx_adiff
     environment:
       - API_URL=https://api.${OHM_DOMAIN}
-      - PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260106_0350.osm.pbf
+      - PLANET_PBF_URL=https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260408_0307.osm.pbf
       - MINUTE_REPLICATION_URL=https://planet.openhistoricalmap.org/?prefix=replication/minute/
       # Add OSMX_INITIAL_SEQNUM to start from a specific sequence number
-      # - OSMX_INITIAL_SEQNUM=1881020
+      # - OSMX_INITIAL_SEQNUM=1900000
       - AWS_S3_BUCKET=planet.openhistoricalmap.org
     env_file:
       - ./.env.osmcha
@@ -22,5 +22,5 @@ services:
       - ${PWD}/images/ohmx-adiff-builder/update.sh:/app/update.sh
     networks:
       - ohm_network
-    cpus: '8.0'
-    mem_limit: 20G
+    # cpus: '8.0'
+    # mem_limit: 20G

hetzner/overpass/overpass.base.yml

@@ -9,10 +9,10 @@ services:
     environment:
       OVERPASS_META: attic
       OVERPASS_MODE: init
-      OVERPASS_PLANET_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260216_0301.osm.pbf
+      OVERPASS_PLANET_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-260408_0307.osm.pbf
       OVERPASS_DIFF_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute
       OVERPASS_RULES_LOAD: 10
-      OVERPASS_REPLICATION_SEQUENCE_NUMBER: 1796000
+      OVERPASS_REPLICATION_SEQUENCE_NUMBER: 1900000
       OVERPASS_ALLOW_DUPLICATE_QUERIES: yes
     restart: always
     ## Disable healthcheck during initialization phase to prevent premature restarts

hetzner/overpass/overpass.production.yml

@@ -1,4 +1,4 @@
 services:
   overpass:
-    mem_limit: 16g
-    cpus: "8.0"
+    # mem_limit: 16g
+    # cpus: "8.0"

hetzner/tiler/config/postgresql.production.conf

@@ -2,18 +2,18 @@
 # CONNECTIONS AND AUTHENTICATION
 #------------------------------------------------------------------------------
 listen_addresses = '*'                    # Allow connections from any network interface
-max_connections = 200                      # Increase if you expect more concurrent connections
+max_connections = 300                      # 4 Tegola replicas × 56 conn + refresh/imposm headroom
 superuser_reserved_connections = 5         # Reserve connections for superusers
 
 #------------------------------------------------------------------------------
 # RESOURCE USAGE
 #------------------------------------------------------------------------------
 
 # - Memory Configuration -
-shared_buffers = 10GB                      # ~25% of 40GB container limit
-work_mem = 256MB                           # Memory for each sort/hash operation; be cautious with many parallel queries
-maintenance_work_mem = 4GB                 # Larger memory for VACUUM / CREATE INDEX / ALTER
-effective_cache_size = 26GB                # ~65% of 40GB container limit
+shared_buffers = 16GB                      # ~13% of 125GB RAM
+work_mem = 256MB                           # Memory for each sort/hash operation; balanced for 300 max connections
+maintenance_work_mem = 8GB                 # Larger memory for VACUUM / CREATE INDEX / ALTER / REFRESH
+effective_cache_size = 80GB                # ~65% of 125GB RAM
 
 # - Disk Optimization for SSD (if using SSD) -
 random_page_cost = 1.0                     # Lower cost for random I/O on SSD
@@ -32,7 +32,7 @@ synchronous_commit = off                   # Improves write performance, risk of
 #------------------------------------------------------------------------------
 # AUTOVACUUM SETTINGS
 #------------------------------------------------------------------------------
-autovacuum_max_workers = 6                 # More parallel vacuum workers for busy systems
+autovacuum_max_workers = 8                 # More parallel vacuum workers for busy systems
 autovacuum_naptime = 30s                   # How often the autovacuum daemon checks for work
 autovacuum_vacuum_cost_limit = -1          # Let PostgreSQL adjust vacuum cost dynamically
 
@@ -42,26 +42,27 @@ autovacuum_vacuum_cost_limit = -1          # Let PostgreSQL adjust vacuum cost d
 effective_io_concurrency = 300             # For SSD; helps the planner estimate IO concurrency
 parallel_tuple_cost = 0.001                # Lower cost to encourage parallelization
 parallel_setup_cost = 100                  # Lower to encourage more parallel plans
-max_worker_processes = 25                  # Match 25 CPUs container limit
-max_parallel_workers_per_gather = 6        # Max workers that can help a single query
-max_parallel_workers = 25                  # Total number of parallel workers across all queries
+max_worker_processes = 36                  # Match 36 CPUs
+max_parallel_workers_per_gather = 8        # Max workers that can help a single query
+max_parallel_workers = 36                  # Total number of parallel workers across all queries
 
 #------------------------------------------------------------------------------
 # LOGGING
 #------------------------------------------------------------------------------
 logging_collector = off                   # Disable log collection
+log_min_messages = fatal                   # Only log FATAL and PANIC (crashes, OOM kills)
+log_min_error_statement = panic            # Do not log SQL statements even on errors
 log_statement = 'none'                     # Do not log any statements
 log_duration = off                         # Disable logging query duration
 log_min_duration_statement = -1            # Disable logging slow queries
-log_error_verbosity = terse                # Minimal error messages
+log_error_verbosity = default              # Show enough detail for debugging errors
 log_autovacuum_min_duration = -1           # Do not log autovacuum runs
-log_connections = on                      # Do not log new connections
-log_disconnections = on                   # Do not log disconnections
+log_connections = off                      # Do not log connections
+log_disconnections = off                   # Do not log disconnections
 log_lock_waits = off                       # Do not log lock waits
 log_temp_files = -1                        # Do not log temporary file creation
 log_checkpoints = off                      # Do not log checkpoints
 log_replication_commands = off             # Do not log replication-related commands
-log_directory = '/dev/null'                # Redirect logs to /dev/null (no storage)
 #------------------------------------------------------------------------------
 # CLIENT CONNECTION DEFAULTS
 #------------------------------------------------------------------------------

hetzner/tiler/config/postgresql.staging.conf

@@ -11,10 +11,10 @@ superuser_reserved_connections = 2       # Reserve a few connections for superus
 #------------------------------------------------------------------------------
 
 # - Memory Configuration -
-shared_buffers = 1GB                     # ~25% of total memory for PostgreSQL caching
-work_mem = 32MB                          # Lower memory per query to fit within 4GB RAM
-maintenance_work_mem = 256MB             # Allocate memory for maintenance tasks like VACUUM
-effective_cache_size = 2GB               # 50% of total memory for query planner caching
+shared_buffers = 4GB                     # Conservative, shares server with production
+work_mem = 256MB                         # Enough for import sorting
+maintenance_work_mem = 4GB               # For CREATE INDEX during import
+effective_cache_size = 16GB              # Conservative estimate sharing with production
 
 # - Disk Optimization for SSD -
 random_page_cost = 1.0                   # Optimize for SSD storage
@@ -35,20 +35,20 @@ synchronous_commit = off                 # Improve performance by reducing commi
 # AUTOVACUUM SETTINGS
 #------------------------------------------------------------------------------
 
-autovacuum_max_workers = 2               # Limit autovacuum workers due to lower memory
+autovacuum_max_workers = 3               # Conservative, shares server with production
 autovacuum_naptime = 1min                # Run autovacuum more frequently
 autovacuum_vacuum_cost_limit = -1        # Allow PostgreSQL to auto-adjust vacuum cost
 
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
-effective_io_concurrency = 100           # Reduce IO concurrency to fit within staging constraints
+effective_io_concurrency = 300           # For SSD
 parallel_tuple_cost = 0.001              # Encourage parallel execution
 parallel_setup_cost = 100                # Lower parallel setup cost
-max_worker_processes = 4                 # Limit worker processes to available CPU cores
-max_parallel_workers_per_gather = 4      # Allow more per-query parallel workers
-max_parallel_workers = 4                 # Total parallel workers for staging
+max_worker_processes = 6                 # Conservative, shares server with production
+max_parallel_workers_per_gather = 4      # Max workers per query
+max_parallel_workers = 6                 # Total parallel workers
 
 #------------------------------------------------------------------------------
 # LOGGING

hetzner/tiler/tiler.production.yml

@@ -10,14 +10,15 @@ services:
       - "54329:5432"
     env_file:
       - .env.tiler
-    mem_limit: 40G
-    cpus: "25.0"
+    mem_limit: 100G
+    cpus: "30.0"
+    shm_size: 16g
     networks:
       - ohm_network
 
   tiler_imposm:
     container_name: tiler_imposm
-    image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.3323.h6950acce
+    image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.3347.h9440d7a7
     volumes:
       - tiler_imposm_data:/mnt/data
     env_file:
@@ -120,7 +121,7 @@ services:
 
   tiler_monitor:
     container_name: tiler_monitor
-    image: ghcr.io/openhistoricalmap/tiler-monitor:0.0.1-0.dev.git.3353.h63df1944
+    image: ghcr.io/openhistoricalmap/tiler-monitor:0.0.1-0.dev.git.3340.h7261c0e9
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       # - ../../images/tiler-monitor:/app
@@ -138,13 +139,13 @@ services:
 volumes:
   tiler_pgdata:
     driver: local
-    name: tiler_db_17_03
+    name: tiler_db
   tiler_imposm_data:
     driver: local
-    name: tiler_imposm_17_03
+    name: tiler_imposm
   tiler_monitor_data:
     driver: local
-    name: tiler_monitor_data
+    name: tiler_monitor_data_v3
 
 networks:
   ohm_network:

hetzner/traefik/traefik.template.yml

@@ -218,7 +218,7 @@ http:
     tiler_monitor:
       loadBalancer:
         servers:
-          - url: http://tiler-monitor:8001
+          - url: http://tiler_monitor:8001
 
 providers:
   file:

images/tiler-imposm/Dockerfile

@@ -40,11 +40,11 @@ RUN git clone https://github.com/OpenHistoricalMap/DateFunctions-plpgsql.git /us
 
 WORKDIR /go
 ENV GOPATH /go
-RUN echo "Version v0.14.2"
-RUN git clone https://github.com/omniscale/imposm3.git $GOPATH/src/github.com/omniscale/imposm3
+RUN echo "objs_warnings"
+RUN git clone https://github.com/OpenHistoricalMap/imposm3.git $GOPATH/src/github.com/omniscale/imposm3
 
 WORKDIR $GOPATH/src/github.com/omniscale/imposm3
-RUN git checkout v0.14.2
+RUN git checkout objs_warnings
 
 # Apply fixes or patches (if necessary)
 RUN sed -i '/setMaxFileSize/d' cache/ldb_pre_121.go