tools/mpy-triage: Add database layer and GitHub API wrapper.

pi-anl · pi-anl · commit 33fac1f984a0 · 2026-03-26T13:14:59.000+11:00
Implements db.py (SQLite connection, schema init, sync state, sqlite-vec
loading) and gh.py (gh CLI wrapper with pagination multiparse, rate-limit
backoff, search, and diff fetching). Includes tests for both modules.

Signed-off-by: Andrew Leech &lt;andrew.leech@planetinnovation.com.au&gt;
diff --git a/tools/mpy-triage/data/schema.sql b/tools/mpy-triage/data/schema.sql
@@ -0,0 +1,31 @@
+-- MicroPython Issue Triage Database Schema
+
+-- Issues and PRs
+CREATE TABLE IF NOT EXISTS items (
+    id INTEGER PRIMARY KEY,
+    number INTEGER NOT NULL,
+    repo TEXT NOT NULL DEFAULT 'micropython/micropython',
+    type TEXT NOT NULL,      -- 'issue' or 'pr'
+    title TEXT,
+    body TEXT,
+    author TEXT,
+    state TEXT,              -- open, closed
+    labels TEXT,             -- JSON array of label names
+    created_at TEXT,
+    closed_at TEXT,
+    updated_at TEXT,
+    UNIQUE(repo, number)
+);
+
+-- Sync state for incremental updates
+CREATE TABLE IF NOT EXISTS sync_state (
+    key TEXT PRIMARY KEY,
+    value TEXT
+);
+
+-- Indexes
+CREATE INDEX IF NOT EXISTS idx_items_repo ON items(repo);
+CREATE INDEX IF NOT EXISTS idx_items_type ON items(type);
+CREATE INDEX IF NOT EXISTS idx_items_state ON items(state);
+CREATE INDEX IF NOT EXISTS idx_items_created ON items(created_at);
+CREATE INDEX IF NOT EXISTS idx_items_repo_number ON items(repo, number);
diff --git a/tools/mpy-triage/pyproject.toml b/tools/mpy-triage/pyproject.toml
@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "mpy-triage"
+version = "0.1.0"
+description = "CLI tool for detecting duplicate/related MicroPython issues and PRs"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+
+dependencies = [
+    "sqlite-vec>=0.1.6",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "ruff>=0.4.0",
+]
+
+[project.scripts]
+mpy-triage = "mpy_triage.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 99
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
diff --git a/tools/mpy-triage/src/mpy_triage.egg-info/PKG-INFO b/tools/mpy-triage/src/mpy_triage.egg-info/PKG-INFO
@@ -0,0 +1,10 @@
+Metadata-Version: 2.4
+Name: mpy-triage
+Version: 0.1.0
+Summary: CLI tool for detecting duplicate/related MicroPython issues and PRs
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: sqlite-vec>=0.1.6
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: ruff>=0.4.0; extra == "dev"
diff --git a/tools/mpy-triage/src/mpy_triage.egg-info/SOURCES.txt b/tools/mpy-triage/src/mpy_triage.egg-info/SOURCES.txt
@@ -0,0 +1,13 @@
+pyproject.toml
+src/mpy_triage/__init__.py
+src/mpy_triage/cli.py
+src/mpy_triage/db.py
+src/mpy_triage/gh.py
+src/mpy_triage.egg-info/PKG-INFO
+src/mpy_triage.egg-info/SOURCES.txt
+src/mpy_triage.egg-info/dependency_links.txt
+src/mpy_triage.egg-info/entry_points.txt
+src/mpy_triage.egg-info/requires.txt
+src/mpy_triage.egg-info/top_level.txt
+tests/test_db.py
+tests/test_gh.py
diff --git a/tools/mpy-triage/src/mpy_triage.egg-info/dependency_links.txt b/tools/mpy-triage/src/mpy_triage.egg-info/dependency_links.txt
@@ -0,0 +1 @@
+
diff --git a/tools/mpy-triage/src/mpy_triage.egg-info/entry_points.txt b/tools/mpy-triage/src/mpy_triage.egg-info/entry_points.txt
@@ -0,0 +1,2 @@
+[console_scripts]
+mpy-triage = mpy_triage.cli:main
diff --git a/tools/mpy-triage/src/mpy_triage.egg-info/requires.txt b/tools/mpy-triage/src/mpy_triage.egg-info/requires.txt
@@ -0,0 +1,5 @@
+sqlite-vec>=0.1.6
+
+[dev]
+pytest>=7.0.0
+ruff>=0.4.0
diff --git a/tools/mpy-triage/src/mpy_triage.egg-info/top_level.txt b/tools/mpy-triage/src/mpy_triage.egg-info/top_level.txt
@@ -0,0 +1 @@
+mpy_triage
diff --git a/tools/mpy-triage/src/mpy_triage/__init__.py b/tools/mpy-triage/src/mpy_triage/__init__.py
diff --git a/tools/mpy-triage/src/mpy_triage/cli.py b/tools/mpy-triage/src/mpy_triage/cli.py
@@ -0,0 +1,29 @@
+"""CLI entry point for mpy-triage."""
+
+import argparse
+import logging
+
+
+def main() -> None:
+    """Main entry point for the mpy-triage CLI."""
+    parser = argparse.ArgumentParser(
+        prog="mpy-triage",
+        description="Detect duplicate and related MicroPython issues/PRs using semantic search.",
+    )
+    parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose logging.")
+    parser.add_argument(
+        "--db",
+        default="data/triage.db",
+        help="Path to the SQLite database (default: data/triage.db).",
+    )
+
+    _args = parser.parse_args()
+
+    logging.basicConfig(
+        level=logging.DEBUG if _args.verbose else logging.INFO,
+        format="%(levelname)s: %(message)s",
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/mpy-triage/src/mpy_triage/db.py b/tools/mpy-triage/src/mpy_triage/db.py
@@ -0,0 +1,85 @@
+"""Database layer for mpy-triage.
+
+Manages SQLite connections, schema initialization, sync state, and
+sqlite-vec extension loading.
+"""
+
+import logging
+import sqlite3
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+def get_connection(db_path: Path) -> sqlite3.Connection:
+    """Open an SQLite connection with Row factory and WAL mode.
+
+    Args:
+        db_path: Path to the SQLite database file.
+
+    Returns:
+        An open sqlite3.Connection.
+    """
+    conn = sqlite3.connect(str(db_path))
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    return conn
+
+
+def init_db(conn: sqlite3.Connection, schema_path: Path) -> None:
+    """Read and execute a schema SQL file against the connection.
+
+    Safe to call multiple times (uses IF NOT EXISTS in schema).
+
+    Args:
+        conn: An open sqlite3 connection.
+        schema_path: Path to the .sql schema file.
+    """
+    schema_path = Path(schema_path)
+    schema_sql = schema_path.read_text()
+    conn.executescript(schema_sql)
+    conn.commit()
+
+
+def get_sync_state(conn: sqlite3.Connection, key: str) -> str | None:
+    """Retrieve a value from the sync_state table.
+
+    Args:
+        conn: An open sqlite3 connection.
+        key: The sync state key to look up.
+
+    Returns:
+        The stored value string, or None if the key is not found.
+    """
+    cursor = conn.execute("SELECT value FROM sync_state WHERE key = ?", (key,))
+    row = cursor.fetchone()
+    return row[0] if row else None
+
+
+def set_sync_state(conn: sqlite3.Connection, key: str, value: str) -> None:
+    """Insert or update a value in the sync_state table.
+
+    Args:
+        conn: An open sqlite3 connection.
+        key: The sync state key.
+        value: The value to store.
+    """
+    conn.execute(
+        "INSERT OR REPLACE INTO sync_state (key, value) VALUES (?, ?)",
+        (key, value),
+    )
+    conn.commit()
+
+
+def load_vec_extension(conn: sqlite3.Connection) -> None:
+    """Load the sqlite-vec extension into the connection.
+
+    Args:
+        conn: An open sqlite3 connection.
+    """
+    import sqlite_vec
+
+    conn.enable_load_extension(True)
+    sqlite_vec.load(conn)
+    conn.enable_load_extension(False)
+    log.info("sqlite-vec extension loaded")
diff --git a/tools/mpy-triage/src/mpy_triage/gh.py b/tools/mpy-triage/src/mpy_triage/gh.py
@@ -0,0 +1,167 @@
+"""GitHub API wrapper for mpy-triage.
+
+Uses the ``gh`` CLI tool for authenticated access. Handles pagination,
+rate-limit back-off, and search queries.
+"""
+
+import json
+import logging
+import re
+import subprocess
+import time
+from urllib.parse import quote
+
+log = logging.getLogger(__name__)
+
+# Rate limiting: 5000 requests/hour for authenticated users.
+REQUESTS_PER_HOUR = 5000
+REQUEST_DELAY: float = 3600 / REQUESTS_PER_HOUR  # ~0.72s
+_last_request_time: float = 0.0
+
+
+def gh_api(
+    endpoint: str,
+    *,
+    paginate: bool = False,
+    accept: str = "application/vnd.github+json",
+    method: str | None = None,
+) -> list | dict | str | None:
+    """Call the GitHub API via the ``gh`` CLI.
+
+    Args:
+        endpoint: API endpoint path (e.g. ``repos/owner/repo/issues``).
+        paginate: If True, pass ``--paginate`` and merge concatenated JSON arrays.
+        accept: Value for the Accept header.
+        method: HTTP method override (e.g. ``GET``, ``POST``). Omitted when None.
+
+    Returns:
+        Parsed JSON (list or dict), raw text for non-JSON accept types, or
+        None on error.
+    """
+    global _last_request_time
+
+    cmd = ["gh", "api", "-H", f"Accept: {accept}"]
+    if paginate:
+        cmd.append("--paginate")
+    if method is not None:
+        cmd.extend(["--method", method])
+    cmd.append(endpoint)
+
+    while True:
+        # Enforce minimum delay between requests to stay under rate limit.
+        elapsed = time.time() - _last_request_time
+        if elapsed < REQUEST_DELAY:
+            time.sleep(REQUEST_DELAY - elapsed)
+
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        _last_request_time = time.time()
+
+        if result.returncode != 0:
+            # Check for rate limiting (HTTP 403 or 429).
+            stderr = result.stderr
+            if "403" in stderr or "429" in stderr:
+                reset_ts = _parse_rate_limit_reset(stderr)
+                if reset_ts is not None:
+                    wait = max(0, reset_ts - time.time()) + 1
+                    log.warning("Rate limited. Sleeping %.0f seconds until reset.", wait)
+                    time.sleep(wait)
+                    continue
+            log.error("gh api error (rc=%d): %s", result.returncode, result.stderr)
+            return None
+
+        break
+
+    if not result.stdout.strip():
+        return [] if paginate else None
+
+    # Non-JSON responses (e.g. diff).
+    if "json" not in accept:
+        return result.stdout
+
+    # Paginated responses: gh --paginate concatenates multiple JSON arrays.
+    if paginate:
+        return _parse_concatenated_json(result.stdout)
+
+    return json.loads(result.stdout)
+
+
+def gh_search(query: str, *, date_range: tuple[str, str] | None = None) -> list[dict]:
+    """Search GitHub issues/PRs via the search API.
+
+    Args:
+        query: Search query string (e.g. ``repo:micropython/micropython is:issue``).
+        date_range: Optional (start, end) date strings in ``YYYY-MM-DD`` format.
+            Adds a ``created:start..end`` qualifier to the query.
+
+    Returns:
+        List of item dicts from the search results.
+    """
+    q = query
+    if date_range is not None:
+        start, end = date_range
+        q += f" created:{start}..{end}"
+
+    endpoint = f"search/issues?q={quote(q)}&per_page=100&sort=updated&order=desc"
+    result = gh_api(endpoint)
+    if result is None or not isinstance(result, dict):
+        return []
+    return result.get("items", [])
+
+
+def gh_diff(repo: str, pr_number: int) -> str | None:
+    """Fetch the diff for a pull request.
+
+    Args:
+        repo: Repository in ``owner/name`` format.
+        pr_number: PR number.
+
+    Returns:
+        The diff as a string, or None on error.
+    """
+    result = gh_api(
+        f"repos/{repo}/pulls/{pr_number}",
+        accept="application/vnd.github.diff",
+    )
+    if isinstance(result, str):
+        return result
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _parse_concatenated_json(text: str) -> list:
+    """Parse potentially concatenated JSON arrays from ``gh --paginate``.
+
+    When paginating, ``gh`` concatenates raw JSON responses without a
+    separator.  Two consecutive arrays like ``[1,2][3,4]`` must be merged
+    into ``[1,2,3,4]``.
+    """
+    data: list = []
+    decoder = json.JSONDecoder()
+    text = text.strip()
+    pos = 0
+    while pos < len(text):
+        try:
+            obj, end = decoder.raw_decode(text, pos)
+            if isinstance(obj, list):
+                data.extend(obj)
+            else:
+                data.append(obj)
+            pos = end
+            # Skip whitespace between concatenated values.
+            while pos < len(text) and text[pos] in " \t\n\r":
+                pos += 1
+        except json.JSONDecodeError:
+            break
+    return data
+
+
+def _parse_rate_limit_reset(stderr: str) -> float | None:
+    """Extract ``X-RateLimit-Reset`` epoch timestamp from gh stderr output."""
+    match = re.search(r"X-RateLimit-Reset:\s*(\d+)", stderr, re.IGNORECASE)
+    if match:
+        return float(match.group(1))
+    return None
diff --git a/tools/mpy-triage/tests/__init__.py b/tools/mpy-triage/tests/__init__.py
diff --git a/tools/mpy-triage/tests/test_db.py b/tools/mpy-triage/tests/test_db.py
diff --git a/tools/mpy-triage/tests/test_gh.py b/tools/mpy-triage/tests/test_gh.py
diff --git a/tools/mpy-triage/uv.lock b/tools/mpy-triage/uv.lock

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+[console_scripts]`
	`2`	`+mpy-triage = mpy_triage.cli:main`