remove database access when not needed, put some volumes in read only in docker-compose.yml, make minio console work

Author: Obada Haddad

.env_circleci

@@ -41,5 +41,6 @@ SUBMISSIONS_API_URL=http://django:8000/api
 # Nginx settings
 # -----------------------------------------------------------------------------
 HTTPS=False
-RATE_LIMIT=10000
+RATE_LIMIT=100
 DOMAIN_NAME=localhost
+DATABASE_ACCESS=True

.env_sample

@@ -1,17 +1,23 @@
-SECRET_KEY=change-this-secret
+# Use openssl rand -hex 32 to generate this secret key, or generate it however you want and copy it here
+SECRET_KEY=
 
 # For local setup and debug
 DEBUG=False
 
+# -----------------------------------------------------------------------------
 # Database
+# -----------------------------------------------------------------------------
 DB_HOST=db
 DB_NAME=postgres
 DB_USERNAME=postgres
 DB_PASSWORD=postgres
 DB_PORT=5432
 
+# -----------------------------------------------------------------------------
+# Django
+# -----------------------------------------------------------------------------
 DJANGO_SETTINGS_MODULE=settings.develop
-ALLOWED_HOSTS=localhost,example.com
+ALLOWED_HOSTS=localhost,
 SUBMISSIONS_API_URL=http://django:8000/api
 MAX_EXECUTION_TIME_LIMIT=600 # time limit for the default queue (in seconds)
 
@@ -21,24 +27,29 @@ MAX_EXECUTION_TIME_LIMIT=600 # time limit for the default queue (in seconds)
 HTTPS=False
 RATE_LIMIT=5
 DOMAIN_NAME=localhost
-
-
-# SSL style domain definition
 TLS_EMAIL=your@email.com
-# DOMAIN_NAME=example.com:443
 
+# -----------------------------------------------------------------------------
+# RabbitMQ
+# -----------------------------------------------------------------------------
 RABBITMQ_HOST=rabbit
 RABBITMQ_DEFAULT_USER=rabbit-username
 RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change
 RABBITMQ_MANAGEMENT_PORT=15672
 RABBITMQ_PORT=5672
 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds
 
-FLOWER_PUBLIC_PORT=5555
 
+# -----------------------------------------------------------------------------
+# Flower
+# -----------------------------------------------------------------------------
+FLOWER_PUBLIC_PORT=5555
 FLOWER_BASIC_AUTH=root:password-you-should-change
 
-SELENIUM_HOSTNAME=selenium
+
+# -----------------------------------------------------------------------------
+# Email Settings
+# -----------------------------------------------------------------------------
 
 # Uncomment to enable email settings
 #EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend
@@ -76,21 +87,6 @@ AWS_QUERYSTRING_AUTH=False
 #WORKER_BUNDLE_URL_REWRITE=http://localhost:9000|http://minio:9000
 
 
-# -----------------------------------------------------------------------------
-# Limit for re-running submission
-# This is used to limit users to rerun submissions
-# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT
-# -----------------------------------------------------------------------------
-RERUN_SUBMISSION_LIMIT=30
-
-
-# -----------------------------------------------------------------------------
-# Enable or disbale regular email sign-in an sign-up
-# -----------------------------------------------------------------------------
-ENABLE_SIGN_UP=True
-ENABLE_SIGN_IN=True
-
-
 # # S3 storage example
 # STORAGE_TYPE=s3
 # AWS_ACCESS_KEY_ID=12312312312312312331223
@@ -113,6 +109,20 @@ ENABLE_SIGN_IN=True
 # GS_PRIVATE_BUCKET_NAME=private
 # GOOGLE_APPLICATION_CREDENTIALS=/app/certs/google-storage-api.json
 
+# -----------------------------------------------------------------------------
+# Limit for re-running submission
+# This is used to limit users to rerun submissions
+# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT
+# -----------------------------------------------------------------------------
+RERUN_SUBMISSION_LIMIT=30
+
+
+# -----------------------------------------------------------------------------
+# Enable or disbale regular email sign-in an sign-up
+# -----------------------------------------------------------------------------
+ENABLE_SIGN_UP=True
+ENABLE_SIGN_IN=True
+
 
 # -----------------------------------------------------------------------------
 # Logging (Serialized outputs the logs in JSON format)

docker-compose.yml

@@ -3,16 +3,16 @@ services:
   #   Web Services
   #----------------------------------------------------------------------------------------------------
   nginx:
-    image: nginx:latest
+    image: nginx:alpine
     env_file: .env
     environment:
       - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
     command: ["nginx", "-g", "daemon off;"]
     volumes:
-      - ./src/staticfiles:/var/www/django/static
-      - ./maintenance_mode/:/srv
+      - ./src/staticfiles:/var/www/django/static:ro
+      - ./maintenance_mode/:/srv:ro
+      - ./nginx/:/etc/nginx/templates/:ro
       - ./nginx/certs/:/var/cache/nginx/acme-letsencrypt
-      - ./nginx/:/etc/nginx/templates/
       - ./var/log/nginx/:/var/log/nginx
     restart: unless-stopped
     ports:
@@ -22,14 +22,8 @@ services:
       - 5432:5432
       - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672
       - ${RABBITMQ_PORT}:5672
-      - ${MINIO_PORT}:9000
+      - ${MINIO_PORT:-9000}:9000
       - ${FLOWER_PUBLIC_PORT:-5555}:5555
-    depends_on:
-      - django
-      - minio
-      - rabbit
-      - site_worker
-      - db
     networks:
       - frontend
       - backend
@@ -65,6 +59,7 @@ services:
         max-file: "5"
     networks:
       - backend
+      - frontend
 
 
   #----------------------------------------------------------------------------------------------------
@@ -81,7 +76,7 @@ services:
       - 9001
     env_file: .env
     environment:
-      MINIO_BROWSER_REDIRECT_URL: "http://localhost/console"
+      MINIO_BROWSER_REDIRECT_URL: "http://${DOMAIN_NAME}/console"
     healthcheck:
       test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"]
       interval: 5s
@@ -150,7 +145,7 @@ services:
     volumes:
       - ./var/postgres:/var/lib/postgresql/18/:delegated
       - ./backups:/app/backups
-      - ./my-postgres.conf:/etc/postgresql/postgresql.conf
+      - ./my-postgres.conf:/etc/postgresql/postgresql.conf:ro
     restart: unless-stopped
     logging:
       options:
@@ -260,7 +255,7 @@ services:
       - django
       - rabbit
     volumes:
-      - ./compute_worker:/app
+      - ./compute_worker:/app:ro
       - ${HOST_DIRECTORY:-/tmp/codabench}:/codabench
       # Actual connection back to docker parent to run things
       - /var/run/docker.sock:/var/run/docker.sock
@@ -278,6 +273,7 @@ services:
         max-file: "5"
     networks:
       - backend
+      - frontend
 
 networks:
   frontend:

nginx/extra/anti_scrapper.conf.template

@@ -2,11 +2,6 @@
 limit_req_status 429;
 limit_req zone=scraperlimit burst=25 nodelay;
 
-location /robots.txt {
-    autoindex off;
-    alias /etc/nginx/templates/robots.txt;
-}
-
 # Deny access to bots
 # Block user agents that tend to be scrapers and badly behaved bots
 if ($bad_bot = 1) {

nginx/http/minio.conf.template

@@ -10,11 +10,37 @@ server {
     proxy_request_buffering off;
 
     location / {
+        include extra/anti_scrapper.conf;
+
         # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
         proxy_http_version 1.1;
         proxy_set_header Connection "";
         chunked_transfer_encoding off;
 
         proxy_pass http://minio:9000;
     }
+
+    # MinIO console
+    location /console/ {
+        include extra/anti_scrapper.conf;
+
+        rewrite ^/console/(.*)$ /$1 break;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-NginX-Proxy true;
+
+        # This is necessary to pass the correct IP to be hashed
+        real_ip_header X-Real-IP;
+        proxy_connect_timeout 300;
+
+        # To support websocket
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "Upgrade";
+
+        chunked_transfer_encoding off;
+        proxy_pass http://minio:9001;
+    }
 }