system status folder (#29)

* everything for system status

* without coin-api

Author: vineeshah

status/docker-compose.yml

@@ -0,0 +1,34 @@
+version: '2'
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    restart: always
+    ports:
+      - 9090:9090
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+
+  scraper:
+    container_name: scraper
+    build:
+      context: .
+      dockerfile: ./dockerfile.scraper
+    volumes:
+      - ./output:/app/output
+      - ./templates:/app/templates
+      - ./json.json:/app/json.json
+    ports:
+      - "8000:8000"
+    restart: 'on-failure'
+    command: python3 scraper.py --json json.json --promurl "http://one.sce/prometheus/metrics"
+
+
+  nginx:
+    image: nginx:1.25.3
+    ports:
+      - 80:80
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf

status/dockerfile.scraper

@@ -0,0 +1,14 @@
+# filepath: /Users/vineet/Projects/sce_system_status/prom/Dockerfile.scraper
+FROM python:3.9-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip3 install -r requirements.txt
+
+COPY flags.py .
+COPY scraper.py .
+COPY templates/ ./templates/
+
+CMD ["python3", "scraper.py", "--json", "json.json"]

status/flags.py

@@ -0,0 +1,31 @@
+import argparse
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--interval",
+        "-int",
+        type= int,
+        default = 15,
+        help = "interval for how often queries should be done"
+    )
+    parser.add_argument(
+        "--port",
+        type = int,
+        default = 8000,
+        help = "port for server to be hosted on, defaults to 8000"
+    )
+    parser.add_argument(
+        "--json",
+        type = str,
+        required = True,
+        help = "argument to a json file, where the json file specifies what services we need to query"
+    )
+    parser.add_argument(
+        "--promurl",
+        type = str,
+        default= "http://prometheus:9090",
+        help = "the url for the promtheus container thats running that has to be scraped"
+    )
+
+    return parser.parse_args()

status/json.json

@@ -0,0 +1,6 @@
+[
+    {
+      "job-id": "prometheus-aggregation",
+      "query": "up"
+    }
+]

status/nginx.conf

@@ -0,0 +1,31 @@
+http {
+    # Define cache path and parameters
+    proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=html_cache:10m max_size=100m inactive=60m;
+    proxy_temp_path /var/cache/nginx/temp;
+    
+    #for http://localhost/status
+    server{
+        listen 80;
+        server_name _;
+        
+        # Enable caching
+        proxy_cache html_cache;
+        proxy_cache_valid 200 302 30s;
+        proxy_cache_valid 404 30s;
+        
+        location /{
+            proxy_pass http://scraper:8000;
+            
+            # Cache HTML files
+            location ~* \.html$ {
+                proxy_pass http://scraper:8000;
+                proxy_cache html_cache;
+                proxy_cache_min_uses 1;
+                proxy_cache_lock on;
+                add_header X-Cache-Status $upstream_cache_status;
+            }
+        }
+    }
+}
+
+events{ }

status/prometheus.yml

@@ -0,0 +1,8 @@
+global:
+  scrape_interval: 10s
+  
+
+scrape_configs:
+  - job_name: 'coin-api'
+    static_configs:
+      - targets: ['coin-api:5000']

status/requirements.txt

@@ -0,0 +1,6 @@
+fastapi==0.84.0
+uvicorn==0.18.3
+Jinja2==3.0.2
+py-grpc-prometheus==0.7.0
+prometheus_api_client
+

status/scraper.py

@@ -0,0 +1,170 @@
+from dataclasses import dataclass
+from fastapi.responses import HTMLResponse
+from prometheus_api_client import PrometheusConnect
+from fastapi import FastAPI, Request
+from fastapi.templating import Jinja2Templates
+from fastapi.staticfiles import StaticFiles
+import uvicorn 
+from flags import get_args
+import json
+import time
+import threading
+import os
+from datetime import datetime, timedelta
+import pytz
+import requests
+
+
+
+app = FastAPI()
+
+pacific_tz = pytz.timezone('US/Pacific')
+
+templates = Jinja2Templates(directory="templates")
+
+args = get_args()
+
+prom = PrometheusConnect(url = args.promurl, disable_ssl=True)#this will query "http://prometheus:9090/api/v1/query?query=up"
+
+metrics_data = []
+up_hours = 24  
+
+@dataclass
+class metrics:
+    job_name: str
+    timestamp: float
+    value: float
+
+def check_status(query):
+    params = {"query" : query}
+    try:
+        response = requests.get("http://prometheus:9090/api/v1/query", params = params)
+        response.raise_for_status()# Raise an error for HTTP issues
+        json_response = response.json()
+        if json_response["status"]=="success":
+            return True
+        elif json_response["status"]==None:
+            print("the status key does not exist!")
+            return False
+        else:
+            return False
+
+    except Exception as e:
+        print(f"Error querying Prometheus: {e}")
+        return None
+
+def polling_loop(interval, config):
+        while True: 
+            global metrics_data
+            metrics_data = []
+            for hosts in config:
+                service_name = hosts["job-id"]
+                prom_query = hosts["query"]
+                if prom_query == "up":  
+                    process_up_query(prom_query, service_name)  
+            time.sleep(interval)
+
+service_data = {}
+
+def process_up_query(query, service_name):
+    global metrics_data, service_data
+    process_time_query("time() - process_start_time_seconds", service_name)
+    if not check_status(query="up"):
+        print("status is not success, please look into it!!")
+    try:
+        result = prom.custom_query(query=query)
+        if not result:
+            print(f"No results for query: {query}")
+            last_active = datetime.now(pacific_tz).strftime("%Y-%m-%d %H:%M:%S %Z")
+            metrics_data.append({
+                "instance": service_name,
+                "status": "Error in querying"
+            })
+            return
+        
+        for metric in result:
+            job_name = metric.get('metric',{}).get('job', "")#for later use in dataclass
+            time_stamp = metric.get('value', [])[0]#for later use in dataclass
+            value = metric.get('value', [])[1]
+            # last_active = datetime.now(pacific_tz).strftime("%Y-%m-%d %H:%M:%S %Z")
+            status = "Healthy" if float(value) > 0 else "Unhealthy"
+            if status == "Unhealthy":
+                current = get_first_match_time(prom=prom, prom_query="up", match_value=0, hours=up_hours)
+                metrics_data.append({
+                    "instance": service_name,
+                    "status": current
+                })
+            else:
+                metrics_data.append({
+                    "instance": service_name,
+                    "status": "Healthy"
+                })
+    except Exception as e:
+        print(f"Error processing query '{query}': {e}")
+        metrics_data.append({
+            "instance": service_name,
+            "status": "Unhealthy due to error!"
+        })
+
+
+def process_time_query(query, service_name):
+    global metrics_data, up_hours
+    try:
+        result = prom.custom_query(query=query)
+        if result and len(result) > 0:
+            first_result = result[0]
+            uptime_seconds = float(first_result["value"][1])
+            up_hours = int(uptime_seconds/3600)
+            if up_hours == 0:
+                up_hours = 1
+    except Exception as e:
+        print(f"Error processing time query '{query}': {e}")
+
+def get_first_match_time(prom, prom_query, match_value=0, hours=24):
+    global metrics_data
+    prom_query = "up"
+    start_time = datetime.now() - timedelta(hours=hours)
+    end_time = datetime.now() 
+
+    try:
+        result = prom.get_metric_range_data(
+            metric_name=prom_query,
+            start_time=start_time,
+            end_time=end_time,
+        )
+
+        for series in result:
+            saw_up = False
+            for timestamp, value in reversed(series["values"]):
+                v = float(value)
+                if v == 1:
+                    saw_up = True
+                elif v == 0 and saw_up:
+                    utc_time = datetime.utcfromtimestamp(float(timestamp))
+                    pacific_time = utc_time.astimezone(pacific_tz)
+                    readable_time = pacific_time.strftime("%Y-%m-%d %H:%M:%S %Z")
+                    status = f"Unhealthy as of {readable_time}"
+                    return status
+    except Exception as e:
+        print(f"Error in get_first_match_time: {e}")
+        return "Error checking status history"
+
+
+@app.get("/", response_class=HTMLResponse)
+async def get_metrics(request: Request):
+    return templates.TemplateResponse(
+        "health.html", 
+        {"request": request, "metrics": metrics_data, "timestamp": datetime.now(pacific_tz).strftime("%Y-%m-%d %H:%M:%S %Z")}
+    )
+
+def main():
+    with open(args.json, "r") as file:
+        config = json.load(file)
+
+    polling_thread = threading.Thread(target = polling_loop, args = (args.interval,config), daemon=True)#The daemon=True ensures the thread exits when the main program exits.
+    polling_thread.start()
+
+    uvicorn.run(app, host="0.0.0.0", port=args.port)
+
+if __name__ == "__main__":
+    main()

status/templates/health.html

@@ -0,0 +1,55 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Prometheus Metrics</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 20px;
+        }
+        h1 {
+            color: #333;
+        }
+        table {
+            border-collapse: collapse;
+            width: 100%;
+            margin-top: 20px;
+        }
+        th, td {
+            border: 1px solid #ddd;
+            padding: 8px;
+            text-align: left;
+        }
+        th {
+            background-color: #f2f2f2;
+        }
+        .healthy {
+            color: green;
+        }
+        .tie {
+            color: blue;
+        }
+        .unhealthy {
+            color: red;
+        }
+    </style>
+</head>
+<body>
+    <h1>Prometheus Metrics</h1>
+    <p>Last updated: {{ timestamp }}</p>
+    <table>
+        <tr>
+            <th>Instance</th>
+            <th>Value</th>
+        </tr>
+        {% for item in metrics %}
+        <tr>
+            <td>{{ item.instance }}</td>
+            <td class="{% if item.status == 'Healthy' %}healthy{% else %}unhealthy{% endif %}">
+                {{ item.status }}
+            </td>
+        </tr>
+        {% endfor %}
+    </table>
+</body>
+</html>