replace caddy with nginx allowing us to route all connection through it (rabbit,minio etc)

Author: Obada Haddad

.env_circleci

@@ -8,12 +8,13 @@ DB_PORT=5432
 
 RABBITMQ_DEFAULT_USER=rabbit-username
 RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change
+RABBITMQ_MANAGEMENT_PORT=15672
 RABBITMQ_PORT=5672
 RABBITMQ_HOST=rabbit
 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds
 
 FLOWER_BASIC_AUTH=root:password-you-should-change
-
+FLOWER_PUBLIC_PORT=5555
 DJANGO_SETTINGS_MODULE=settings.test
 
 # Minio local storage example
@@ -28,11 +29,17 @@ AWS_SECRET_ACCESS_KEY=testsecret
 AWS_STORAGE_BUCKET_NAME=public
 AWS_STORAGE_PRIVATE_BUCKET_NAME=private
 # NOTE! port 9000 here should match $MINIO_PORT
-AWS_S3_ENDPOINT_URL=http://172.17.0.1:9000/
+AWS_S3_ENDPOINT_URL=http://minio:9000/
 AWS_QUERYSTRING_AUTH=False
 DJANGO_SUPERUSER_PASSWORD=codabench
 DJANGO_SUPERUSER_EMAIL=test@test.com
 DJANGO_SUPERUSER_USERNAME=codabench
-DOMAIN_NAME=localhost:80
 TLS_EMAIL=your@email.com
-SUBMISSIONS_API_URL=http://django:8000/api
+SUBMISSIONS_API_URL=http://django:8000/api
+
+# -----------------------------------------------------------------------------
+# Nginx settings
+# -----------------------------------------------------------------------------
+HTTPS=False
+RATE_LIMIT=100
+DOMAIN_NAME=localhost

.env_sample

@@ -1,7 +1,7 @@
 SECRET_KEY=change-this-secret
 
 # For local setup and debug
-DEBUG=True
+DEBUG=False
 
 # Database
 DB_HOST=db
@@ -15,8 +15,13 @@ ALLOWED_HOSTS=localhost,example.com
 SUBMISSIONS_API_URL=http://django:8000/api
 MAX_EXECUTION_TIME_LIMIT=600 # time limit for the default queue (in seconds)
 
-# Local domain definition
-DOMAIN_NAME=localhost:80
+# -----------------------------------------------------------------------------
+# Nginx settings
+# -----------------------------------------------------------------------------
+HTTPS=False
+RATE_LIMIT=5
+DOMAIN_NAME=localhost
+
 
 # SSL style domain definition
 TLS_EMAIL=your@email.com
@@ -28,9 +33,6 @@ RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change
 RABBITMQ_MANAGEMENT_PORT=15672
 RABBITMQ_PORT=5672
 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds
-#RABBITMQ_HTTP_PROXY=http://proxy-example:3128
-#RABBITMQ_HTTPS_PROXY=http://proxy-example:3128
-#RABBITMQ_NO_PROXY=localhost,172.0.0.0/8
 
 FLOWER_PUBLIC_PORT=5555
 

docker-compose.yml

@@ -2,24 +2,37 @@ services:
   #----------------------------------------------------------------------------------------------------
   #   Web Services
   #----------------------------------------------------------------------------------------------------
-  caddy:
-    image: caddy:2.11.1
+  nginx:
+    image: nginx:latest
     env_file: .env
     environment:
-      - ACME_AGREE=true
+      - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
+    command: ["nginx", "-g", "daemon off;"]
     volumes:
-      - ./Caddyfile:/etc/caddy/Caddyfile
       - ./src/staticfiles:/var/www/django/static
-      - ./caddy_data:/data
-      - ./caddy_config:/config
-      - ./var/log/caddy:/var/log/
       - ./maintenance_mode/:/srv
+      - ./certsNginx:/var/cache/nginx/acme-letsencrypt
+      - ./nginx/:/etc/nginx/templates/
+      - ./var/log/nginx/:/var/log/nginx
     restart: unless-stopped
     ports:
       - 80:80
       - 443:443
+      - 8000:8000
+      - 5432:5432
+      - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672
+      - ${RABBITMQ_PORT}:5672
+      - ${MINIO_PORT}:9000
+      - ${FLOWER_PUBLIC_PORT:-5555}:5555
     depends_on:
       - django
+      - minio
+      - rabbit
+      - site_worker
+      - db
+    networks:
+      - frontend
+      - backend
 
   django:
     container_name: django
@@ -39,7 +52,7 @@ services:
       - ./var/logs:/app/logs
     restart: unless-stopped
     ports:
-      - 8000:8000
+      - 8000
     depends_on:
       - db
       - rabbit
@@ -50,6 +63,8 @@ services:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
 
 
   #----------------------------------------------------------------------------------------------------
@@ -62,12 +77,14 @@ services:
       - ./var/minio:/export
     restart: unless-stopped
     ports:
-      - $MINIO_PORT:9000
+      - 9000
     env_file: .env
     healthcheck:
       test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"]
       interval: 5s
       retries: 5
+    networks:
+      - backend
   createbuckets:
     image: minio/mc:RELEASE.2025-07-21T05-28-08Z
     depends_on:
@@ -92,6 +109,8 @@ services:
       fi;
       exit 0;
       "
+    networks:
+      - backend
 
   #----------------------------------------------------------------------------------------------------
   #   Local development helper, rebuilds RiotJS/Stylus on change
@@ -109,7 +128,6 @@ services:
         max-size: "20m"
         max-file: "5"
 
-
   #----------------------------------------------------------------------------------------------------
   #   Database Service
   #
@@ -125,7 +143,7 @@ services:
       - POSTGRES_PASSWORD=${DB_PASSWORD}
     command: ["postgres", "-c", "log_statement=all", "-c", "log_destination=stderr", "-c", "config_file=/etc/postgresql/postgresql.conf"]
     ports:
-      - 5432:5432
+      - 5432
     volumes:
       - ./var/postgres:/var/lib/postgresql/18/:delegated
       - ./backups:/app/backups
@@ -135,6 +153,8 @@ services:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
 
   #----------------------------------------------------------------------------------------------------
   #   Rabbitmq & Flower monitoring tool
@@ -150,20 +170,18 @@ services:
     # containers being destroyed..!
     hostname: rabbit
     env_file: .env
-    environment:
-      - http_proxy=${RABBITMQ_HTTP_PROXY}
-      - https_proxy=${RABBITMQ_HTTPS_PROXY}
-      - no_proxy=${RABBITMQ_NO_PROXY}
     ports:
-      - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672
-      - ${RABBITMQ_PORT}:5672
+      - 15672
+      - 5672
     volumes:
       - ./var/rabbit:/var/lib/rabbitmq
     restart: unless-stopped
     logging:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
 
   flower:
     container_name: flower
@@ -173,13 +191,15 @@ services:
       - CELERY_BROKER_URL=pyamqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_HOST}:${RABBITMQ_PORT}//
     restart: unless-stopped
     ports:
-      - ${FLOWER_PUBLIC_PORT:-5555}:5555
+      - 5555
     depends_on:
       - rabbit
     logging:
         options:
             max-size: "20m"
             max-file: "5"
+    networks:
+      - backend
 
   #----------------------------------------------------------------------------------------------------
   #   Redis
@@ -188,12 +208,14 @@ services:
     container_name: redis
     image: redis
     ports:
-      - 6379:6379
+      - 6379
     restart: unless-stopped
     logging:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
 
   #----------------------------------------------------------------------------------------------------
   #   Celery Service
@@ -221,6 +243,8 @@ services:
           # Limit memory substantially here so we see any problems that may
           # appear on Heroku ahead of time
           memory: 256M
+    networks:
+      - backend
 
   compute_worker:
     command: ["celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@%n"]
@@ -249,3 +273,10 @@ services:
       options:
         max-size: "20m"
         max-file: "5"
+    networks:
+      - backend
+
+networks:
+  frontend:
+  backend:
+    internal: true

nginx/extra/anti_scrapper.conf.template

@@ -0,0 +1,19 @@
+# Rate limit error code (429) and definition (allow for bursts of 25 requests to load static images etc) and block immediately after the rate limit is applied to an IP
+limit_req_status 429;
+limit_req zone=scraperlimit burst=25 nodelay;
+
+location /robots.txt {
+    autoindex off;
+    alias /etc/nginx/templates/robots.txt;
+}
+
+# Deny access to bots
+# Block user agents that tend to be scrapers and badly behaved bots
+if ($bad_bot = 1) {
+    return 444 "? beep boop ?";
+}
+
+# No scrapers
+if ($scraper = 1) {
+    return 418 "?";
+}

nginx/extra/base_django.conf.template

@@ -0,0 +1,23 @@
+charset utf-8;
+client_max_body_size 10480m;
+client_body_buffer_size 32m;
+sendfile on;
+gzip on;
+
+location ~ /static/(.*) {
+    autoindex off;
+    access_log off;
+    include /etc/nginx/mime.types;
+    root /var/www/django/;
+}
+location ~ /media/(.*) {
+    autoindex off;
+    access_log off;
+    include /etc/nginx/mime.types;
+    root /var/www/django;
+}
+location /favicon.ico {
+    autoindex off;
+    access_log off;
+    alias /var/www/django/static/img/favicon.ico;
+}

nginx/extra/block_map.conf.template

@@ -0,0 +1,33 @@
+limit_req_zone $binary_remote_addr zone=scraperlimit:10m rate=${RATE_LIMIT}r/s;
+
+# Block bad bots
+map $http_user_agent $bad_bot {
+    default 0;
+    ~*(?i)(JikeSpider) 1;
+    ~*(?i)(proximic) 1;
+    ~*(?i)(Sosospider) 1;
+    ~*(?i)(Baiduspider) 1;
+    ~*(?i)(Twitterbot) 1;
+    ~*(?i)(SemrushBot) 1;
+    ~*(?i)(^AIBOT) 1;
+    ~*(?i)(^BunnySlippers) 1;
+    ~*(?i)(^Cegbfeieh) 1;
+    ~*(?i)(^CheeseBot) 1;
+}
+
+# Block scrappers
+map $http_user_agent $scraper {
+    default 0;
+    ~*(?i)(Google-Extended) 1;
+    ~*(?i)(Applebot-Extended) 1;
+    ~*(?i)(anthropic-ai) 1;
+    ~*(?i)(ClaudeBot) 1;
+    ~*(?i)(Claude-Web) 1;
+    ~*(?i)(GPTBot) 1;
+    ~*(?i)(Omgili) 1;
+    ~*(?i)(FacebookBot) 1;
+    ~*(?i)(node-fetch) 1;
+    ~*(?i)(Timpibot) 1;
+    # If you don't provide a User-Agent, you can go away
+    ~*(^-$) 1;
+}

nginx/extra/maintenance.conf.template

@@ -0,0 +1,5 @@
+error_page 503 @maintenance;
+location @maintenance {
+    root /srv;
+    try_files $uri /maintenance.html =503;
+}

nginx/http/flower.conf.template

@@ -0,0 +1,7 @@
+server { 
+    listen ${FLOWER_PUBLIC_PORT};
+    location / {
+        proxy_pass http://flower:5555;
+    }
+    
+}

nginx/http/minio.conf.template

@@ -0,0 +1,32 @@
+server {
+    listen ${MINIO_PORT};
+    # To allow special characters in headers
+    ignore_invalid_headers off;
+    # Allow any size file to be uploaded.
+    # Set to a value such as 1000m; to restrict file size to a specific value
+    client_max_body_size 0;
+    # To disable buffering
+    proxy_buffering off;
+    proxy_request_buffering off;
+
+    location / {
+        # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        chunked_transfer_encoding off;
+
+        proxy_pass http://minio:9000;
+    }
+
+    location /console/ {
+        rewrite ^/console/(.*)$ /$1 break;
+
+        # To support websocket
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+
+        chunked_transfer_encoding off;
+        proxy_pass http://minio;
+    }
+}