everything for system status

vineeshah · vineeshah · commit 8cb8591dd690 · 2025-06-25T12:32:18.000-07:00
diff --git a/status/prom/docker-compose.yml b/status/prom/docker-compose.yml
@@ -0,0 +1,41 @@
+version: '2'
+
+services:
+  coin-api:
+    container_name: coin-api
+    build:
+      context: .
+      dockerfile: ./dockerfile
+    restart: 'on-failure'  
+    ports:
+      - "5000:5000"
+  prometheus:
+    image: prom/prometheus:latest
+    restart: always
+    ports:
+      - 9090:9090
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+
+  scraper:
+    container_name: scraper
+    build:
+      context: .
+      dockerfile: ./dockerfile.scraper
+    volumes:
+      - ./output:/app/output
+      - ./templates:/app/templates
+      - ./json.json:/app/json.json
+    ports:
+      - "8000:8000"
+    restart: 'on-failure'
+    command: python3 scraper.py --json json.json
+
+  nginx:
+    image: nginx:1.25.3
+    ports:
+      - 80:80
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
diff --git a/status/prom/dockerfile b/status/prom/dockerfile
@@ -0,0 +1,14 @@
+FROM python:3.9-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip3 install -r requirements.txt
+
+COPY server.py .
+
+
+EXPOSE 5000
+
+CMD ["python3", "server.py"]
diff --git a/status/prom/dockerfile.scraper b/status/prom/dockerfile.scraper
@@ -0,0 +1,14 @@
+# filepath: /Users/vineet/Projects/sce_system_status/prom/Dockerfile.scraper
+FROM python:3.9-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip3 install -r requirements.txt
+
+COPY flags.py .
+COPY scraper.py .
+COPY templates/ ./templates/
+
+CMD ["python3", "scraper.py", "--json", "json.json"]
diff --git a/status/prom/flags.py b/status/prom/flags.py
@@ -0,0 +1,25 @@
+import argparse
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--interval",
+        "-int",
+        type= int,
+        default = 15,
+        help = "interval for how often queries should be done"
+    )
+    parser.add_argument(
+        "--port",
+        type = int,
+        default = 8000,
+        help = "port for server to be hosted on, defaults to 8000"
+    )
+    parser.add_argument(
+        "--json",
+        type = str,
+        required = True,
+        help = "argument to a json file, where the json file specifies what services we need to query"
+    )
+
+    return parser.parse_args()
diff --git a/status/prom/json.json b/status/prom/json.json
@@ -0,0 +1,13 @@
+[
+    {
+      "hostname": "localhost:3000/metrics",
+      "queries": [
+       
+        {
+            "serviceName": "Coin-API",
+            "query": "up"
+        }
+       
+      ]
+    }
+]
diff --git a/status/prom/nginx.conf b/status/prom/nginx.conf
@@ -0,0 +1,31 @@
+http {
+    # Define cache path and parameters
+    proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=html_cache:10m max_size=100m inactive=60m;
+    proxy_temp_path /var/cache/nginx/temp;
+    
+    #for http://localhost/status
+    server{
+        listen 80;
+        server_name _;
+        
+        # Enable caching
+        proxy_cache html_cache;
+        proxy_cache_valid 200 302 30s;
+        proxy_cache_valid 404 30s;
+        
+        location /{
+            proxy_pass http://scraper:8000;
+            
+            # Cache HTML files
+            location ~* \.html$ {
+                proxy_pass http://scraper:8000;
+                proxy_cache html_cache;
+                proxy_cache_min_uses 1;
+                proxy_cache_lock on;
+                add_header X-Cache-Status $upstream_cache_status;
+            }
+        }
+    }
+}
+
+events{ }
diff --git a/status/prom/output/metrics.html b/status/prom/output/metrics.html
@@ -0,0 +1,22 @@
+
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Prometheus Metrics</title>
+</head>
+<body>
+    <h1>Prometheus Metrics</h1>
+    <table border="1">
+        <tr>
+            <th>Instance</th>
+            <th>Value</th>
+        </tr>
+
+        <tr>
+            <td>coin-api:5000</td>
+            <td>1</td>
+        </tr>
+    
+    </table>
+</body>
+</html>
diff --git a/status/prom/prometheus.yml b/status/prom/prometheus.yml
@@ -0,0 +1,8 @@
+global:
+  scrape_interval: 10s
+  
+
+scrape_configs:
+  - job_name: 'coin-api'
+    static_configs:
+      - targets: ['coin-api:5000']
diff --git a/status/prom/queries.txt b/status/prom/queries.txt
@@ -0,0 +1,21 @@
+{
+      "hostname": "localhost:9090/metrics",
+      "queries": [
+        {
+            "serviceName": "My FastAPI API - CPU Usage",
+            "query": "rate(node_cpu_seconds_total[5m])"
+        },
+        {
+          "serviceName": "Prometheus - Memory Usage",
+          "query": "node_memory_MemAvailable_bytes"
+        },
+        {
+          "serviceName": "My FastAPI API - Uptime",
+          "query": "time() - process_start_time_seconds"
+        },
+        {
+            "serviceName": "Last Down::",
+            "query": "down"
+        }
+      ]
+    }
diff --git a/status/prom/requirements.txt b/status/prom/requirements.txt
@@ -0,0 +1,6 @@
+fastapi==0.84.0
+uvicorn==0.18.3
+Jinja2==3.0.2
+py-grpc-prometheus==0.7.0
+prometheus_api_client
+
diff --git a/status/prom/scraper.py b/status/prom/scraper.py
@@ -0,0 +1,140 @@
+from fastapi.responses import HTMLResponse
+from prometheus_api_client import PrometheusConnect
+from fastapi import FastAPI, Request
+from fastapi.templating import Jinja2Templates
+from fastapi.staticfiles import StaticFiles
+import uvicorn 
+from flags import get_args
+import json
+import time
+import threading
+import os
+from datetime import datetime, timedelta
+import pytz
+
+prom = PrometheusConnect(url = "http://prometheus:9090", disable_ssl=True)#this will query "http://prometheus:9090/api/v1/query?query=up"
+
+app = FastAPI()
+
+pacific_tz = pytz.timezone('US/Pacific')
+
+templates = Jinja2Templates(directory="templates")
+
+args = get_args()
+metrics_data = []
+up_hours = 24  
+
+def polling_loop(interval, config):
+        while True: 
+            global metrics_data
+            metrics_data = []
+            for hosts in config:
+                for query in hosts["queries"]:
+                    service_name = query["serviceName"]
+                    prom_query = query["query"]
+                    if prom_query == "up":  
+                        process_up_query(prom_query, service_name)  
+            time.sleep(interval)
+
+service_data = {}
+
+def process_up_query(query, service_name):
+    global metrics_data, service_data
+    process_time_query("time() - process_start_time_seconds", service_name)
+    
+    try:
+        result = prom.custom_query(query=query)
+        if not result:
+            print(f"No results for query: {query}")
+            last_active = datetime.now(pacific_tz).strftime("%Y-%m-%d %H:%M:%S %Z")
+            metrics_data.append({
+                "instance": service_name,
+                "status": "Error in querying"
+            })
+            return
+        
+        for metric in result:
+            instance = metric['metric'].get('instance', 'unknown')
+            value = metric['value'][1]
+            last_active = datetime.now(pacific_tz).strftime("%Y-%m-%d %H:%M:%S %Z")
+            status = "Healthy" if float(value) > 0 else "Unhealthy"
+            if status == "Unhealthy":
+                current = get_first_match_time(prom=prom, prom_query="up", match_value=0, hours=up_hours)
+                metrics_data.append({
+                    "instance": service_name,
+                    "status": current
+                })
+            else:
+                metrics_data.append({
+                    "instance": service_name,
+                    "status": "Healthy"
+                })
+    except Exception as e:
+        print(f"Error processing query '{query}': {e}")
+        metrics_data.append({
+            "instance": service_name,
+            "status": "Unhealthy due to error!"
+        })
+
+
+def process_time_query(query, service_name):
+    global metrics_data, up_hours
+    try:
+        result = prom.custom_query(query=query)
+        if result and len(result) > 0:
+            first_result = result[0]
+            uptime_seconds = float(first_result["value"][1])
+            up_hours = int(uptime_seconds/3600)
+            if up_hours == 0:
+                up_hours = 1
+    except Exception as e:
+        print(f"Error processing time query '{query}': {e}")
+
+def get_first_match_time(prom, prom_query, match_value=0, hours=24):
+    global metrics_data
+    prom_query = "up"
+    start_time = datetime.now() - timedelta(hours=hours)
+    end_time = datetime.now() 
+
+    try:
+        result = prom.get_metric_range_data(
+            metric_name=prom_query,
+            start_time=start_time,
+            end_time=end_time,
+        )
+
+        for series in result:
+            saw_up = False
+            for timestamp, value in reversed(series["values"]):
+                v = float(value)
+                if v == 1:
+                    saw_up = True
+                elif v == 0 and saw_up:
+                    utc_time = datetime.utcfromtimestamp(float(timestamp))
+                    pacific_time = utc_time.astimezone(pacific_tz)
+                    readable_time = pacific_time.strftime("%Y-%m-%d %H:%M:%S %Z")
+                    status = f"Unhealthy as of {readable_time}"
+                    return status
+    except Exception as e:
+        print(f"Error in get_first_match_time: {e}")
+        return "Error checking status history"
+
+
+@app.get("/", response_class=HTMLResponse)
+async def get_metrics(request: Request):
+    return templates.TemplateResponse(
+        "health.html", 
+        {"request": request, "metrics": metrics_data, "timestamp": datetime.now(pacific_tz).strftime("%Y-%m-%d %H:%M:%S %Z")}
+    )
+
+def main():
+    with open(args.json, "r") as file:
+        config = json.load(file)
+
+    polling_thread = threading.Thread(target = polling_loop, args = (args.interval,config), daemon=True)#The daemon=True ensures the thread exits when the main program exits.
+    polling_thread.start()
+
+    uvicorn.run(app, host="0.0.0.0", port=args.port)
+
+if __name__ == "__main__":
+    main()
diff --git a/status/prom/server.py b/status/prom/server.py
diff --git a/status/prom/templates/health.html b/status/prom/templates/health.html
