Merge pull request #97 from bugout-dev/add-metrics

Add init metrics version.

Author: Andrei-Dolgolev

python/humbug/report.py

@@ -7,6 +7,7 @@
 from dataclasses import dataclass, field
 from enum import Enum
 from functools import wraps
+import json
 
 import logging
 import os
@@ -17,14 +18,30 @@
 from typing import Any, Callable, Dict, List, Optional
 import uuid
 
-import requests
+import requests  # type: ignore
 
+
+from . import utils
 from .consent import HumbugConsent
 from .system_information import (
     SystemInformation,
     generate as generate_system_information,
 )
 
+psutil = None
+
+try:
+    import psutil  # type: ignore
+except ImportError:
+    pass
+
+GPUtil = None
+
+try:
+    import GPUtil  # type: ignore
+except ImportError:
+    pass
+
 
 DEFAULT_URL = "https://spire.bugout.dev"
 
@@ -96,6 +113,9 @@ def __init__(
 
         self.blacklist_fn = blacklist_fn
 
+        self.psutil_exists = psutil is not None
+        self.gputil_exists = GPUtil is not None
+
     def wait(self) -> None:
         concurrent.futures.wait(
             self.report_futures, timeout=float(self.timeout_seconds)
@@ -524,6 +544,65 @@ def showtraceback(*args, **kwargs):
         ipython_shell.showtraceback = showtraceback
         self.setup_excepthook(publish=True, tags=tags)
 
+    def metrics_report(
+        self,
+        cpu: bool = True,
+        gpu: bool = True,
+        memory: bool = True,
+        disk: bool = True,
+        network: bool = True,
+        open_files_flag: bool = True,
+        num_threads_flag: bool = True,
+        processes_flag: bool = True,
+        tags: Optional[List[str]] = None,
+        publish: bool = False,
+        wait: bool = False,
+    ) -> Report:
+        title = "Metrics report"
+
+        metrics: Dict[str, Any] = {}
+
+        if self.gputil_exists:
+            metrics["gpu"] = utils.get_gpu_metrics()
+
+        if self.psutil_exists:
+            if cpu:
+                metrics["cpu"] = utils.get_cpu_metrics()
+
+            if memory:
+                metrics["memory"] = utils.get_memory_metrics()
+
+            if disk:
+                metrics["disk"] = utils.get_disk_metrics()
+
+            if network:
+                metrics["network"] = utils.get_network_metrics()
+
+            if open_files_flag:
+                metrics["open_files"] = utils.get_open_files_metrics()
+
+            if num_threads_flag:
+                metrics["num_threads"] = utils.get_thread_metrics()
+
+            if processes_flag:
+                metrics["processes"] = utils.get_processes_metrics()
+
+        tags = tags if tags is not None else []
+
+        tags.append("type:metrics")
+        tags.extend(self.system_tags())
+
+        report = Report(
+            title=title,
+            tags=tags,
+            content=f"```\n{json.dumps(metrics, indent=4, sort_keys=True)}\n```",
+        )
+
+        if publish:
+            self.publish(report, wait=wait)
+
+        return report
+
 
 class Reporter(HumbugReporter):
     """

python/humbug/utils.py

@@ -0,0 +1,163 @@
+import os
+import types
+import sys
+
+
+def get_cpu_metrics():
+    cpu_metrics = {}
+
+    import psutil
+
+    cpu_metrics["cpu_count"] = psutil.cpu_count()
+    cpu_metrics["cpu_count_logical"] = psutil.cpu_count(logical=True)
+    cpu_metrics["cpu_count_physical"] = psutil.cpu_count(logical=False)
+    cpu_metrics["cpu_percent"] = psutil.cpu_percent()
+    cpu_metrics["cpu_percent_per_core"] = psutil.cpu_percent(percpu=True)
+    current_pid = os.getpid()
+    current_process = psutil.Process(current_pid)
+    cpu_metrics["cpu_load_by_process"] = current_process.cpu_percent()
+    return cpu_metrics
+
+
+def get_gpu_metrics():
+    gpu_metrics = {}
+
+    import GPUtil
+
+    gpus = GPUtil.getGPUs()
+    for gpu_unit in gpus:
+        gpu_id = gpu_unit.id
+        gpu_metrics[f"gpu_{gpu_id}_uuid"] = gpu_unit.uuid
+        gpu_metrics[f"gpu_{gpu_id}_load"] = gpu_unit.load
+        gpu_metrics[f"gpu_{gpu_id}_memory_util_%"] = gpu_unit.memoryUtil
+        gpu_metrics[f"gpu_{gpu_id}_memory_total_MB"] = gpu_unit.memoryTotal
+        gpu_metrics[f"gpu_{gpu_id}_memory_used_MB"] = gpu_unit.memoryUsed
+        gpu_metrics[f"gpu_{gpu_id}_memory_free_MB"] = gpu_unit.memoryFree
+        gpu_metrics[f"gpu_{gpu_id}_driver"] = gpu_unit.driver
+        gpu_metrics[f"gpu_{gpu_id}_name"] = gpu_unit.name
+        gpu_metrics[f"gpu_{gpu_id}_serial"] = gpu_unit.serial
+        gpu_metrics[f"gpu_{gpu_id}_display_mode"] = gpu_unit.display_mode
+        gpu_metrics[f"gpu_{gpu_id}_display_active"] = gpu_unit.display_active
+        gpu_metrics[f"gpu_{gpu_id}_temperature_C"] = gpu_unit.temperature
+    gpu_metrics["gpu_count"] = len(gpus)
+    return gpu_metrics
+
+
+def get_memory_metrics():
+    memory_metrics = {}
+
+    import psutil
+
+    system_memory = psutil.virtual_memory()
+    memory_metrics["total_MB"] = round(system_memory.total / 1024 / 1024, 2)
+    memory_metrics["available_MB"] = round(system_memory.available / 1024 / 1024, 2)
+    memory_metrics["percent_%"] = system_memory.percent
+    memory_metrics["used_MB"] = round(system_memory.used / 1024 / 1024, 2)
+    memory_metrics["free_MB"] = round(system_memory.free / 1024 / 1024, 2)
+    swap = psutil.swap_memory()
+    memory_metrics["swap_total_MB"] = round(swap.total / 1024 / 1024, 2)
+    memory_metrics["swap_used_MB"] = round(swap.used / 1024 / 1024, 2)
+    memory_metrics["swap_free_MB"] = round(swap.free / 1024 / 1024, 2)
+    memory_metrics["swap_percent_%"] = swap.percent
+    return memory_metrics
+
+
+def get_disk_metrics():
+    disk_metrics = {}
+
+    import psutil
+
+    for disk_partition in psutil.disk_partitions():
+        disk_usage = psutil.disk_usage(disk_partition.mountpoint)
+        disk_metrics[str(disk_partition.mountpoint)] = {}
+        disk_metrics[str(disk_partition.mountpoint)]["total_MB"] = round(
+            disk_usage.total / 1024 / 1024, 2
+        )  # in MB
+        disk_metrics[str(disk_partition.mountpoint)]["used_MB"] = round(
+            disk_usage.used / 1024 / 1024, 2
+        )  # in MB
+        disk_metrics[str(disk_partition.mountpoint)]["free_MB"] = round(
+            disk_usage.free / 1024 / 1024, 2
+        )  # in MB
+        disk_metrics[str(disk_partition.mountpoint)]["percent_%"] = disk_usage.percent
+
+    disk_io = psutil.disk_io_counters()
+    disk_metrics["read_count"] = disk_io.read_count
+    disk_metrics["write_count"] = disk_io.write_count
+    disk_metrics["read_MB"] = round(disk_io.read_bytes / 1024 / 1024, 2)  # in MB
+    disk_metrics["write_MB"] = round(disk_io.write_bytes / 1024 / 1024, 2)  # in MB
+    disk_metrics["read_time_s"] = disk_io.read_time / 1000  # in seconds
+    disk_metrics["write_time_s"] = disk_io.write_time / 1000  # in seconds
+    disk_metrics["read_merged_count"] = disk_io.read_merged_count
+    disk_metrics["write_merged_count"] = disk_io.write_merged_count
+    disk_metrics["busy_time_s"] = round(disk_io.busy_time, 2) / 1000  # in seconds
+    return disk_metrics
+
+
+def get_network_metrics():
+    network_metrics = {}
+
+    import psutil
+
+    network_io = psutil.net_io_counters()
+    network_metrics["MB_sent"] = round(network_io.bytes_sent / 1024 / 1024, 2)  # in MB
+    network_metrics["MB_recv"] = round(network_io.bytes_recv / 1024 / 1024, 2)  # in MB
+    network_metrics["packets_sent"] = network_io.packets_sent
+    network_metrics["packets_recv"] = network_io.packets_recv
+    network_metrics["errin"] = network_io.errin
+    network_metrics["errout"] = network_io.errout
+    network_metrics["dropin"] = network_io.dropin
+    network_metrics["dropout"] = network_io.dropout
+
+    return network_metrics
+
+
+def get_open_files_metrics():
+    files_metrics = {}
+
+    import psutil
+
+    open_files = psutil.Process().open_files()
+    files_metrics["total"] = len(open_files)
+
+    return files_metrics
+
+
+def get_thread_metrics():
+    threads_metrics = {}
+
+    import psutil
+
+    threads = psutil.Process().threads()
+    threads_metrics["total"] = len(threads)
+
+    return threads_metrics
+
+
+def get_processes_metrics():
+    process_metrics = {}
+
+    import psutil
+
+    # get all processes memory usage
+
+    processes = psutil.process_iter()
+
+    # Iterate through each process and get its CPU and memory utilization
+    for process in processes:
+        cpu_percent = process.cpu_percent()
+        mem_info = process.memory_info()
+        if process.name() not in process_metrics:
+            process_metrics[f"{process.name()}"] = {
+                "cpu_percent": cpu_percent,
+                "memory_MB": round(mem_info.rss / 1024 / 1024, 2),  # in MB
+                "amount_of_processes": 1,
+            }
+        else:
+            process_metrics[f"{process.name()}"]["cpu_percent"] += cpu_percent
+            process_metrics[f"{process.name()}"]["memory_MB"] += round(
+                mem_info.rss / 1024 / 1024, 2
+            )
+            process_metrics[f"{process.name()}"]["amount_of_processes"] += 1
+
+    return process_metrics

python/mypy.ini

@@ -0,0 +1,4 @@
+[mypy]
+
+[mypy-GPUtil.*]
+ignore_missing_imports = True

python/setup.py

@@ -6,13 +6,10 @@
 
 setup(
     name="humbug",
-    version="0.2.8",
+    version="0.3.0",
     packages=find_packages(),
     package_data={"humbug": ["py.typed"]},
-    install_requires=[
-        "requests",
-        "dataclasses; python_version=='3.6'"
-    ],
+    install_requires=["requests", "dataclasses; python_version=='3.6'"],
     extras_require={
         "dev": [
             "black",
@@ -21,8 +18,10 @@
             "types-pkg_resources",
             "types-requests",
             "types-dataclasses",
+            "types-psutil",
         ],
         "distribute": ["setuptools", "twine", "wheel"],
+        "profile": ["psutil", "GPUtil", "types-psutil"],
     },
     description="Humbug: Do you build developer tools? Humbug helps you know your users.",
     long_description=long_description,