test(priming-group): add real integration tests; add expensive marker

Author: brandon-wada

Makefile

@@ -38,8 +38,8 @@ PYTEST=poetry run pytest -v
 # 	`make test TEST_ARGS="-k some_filter"`
 TEST_ARGS=
 
-CLOUD_FILTERS = -m "not run_only_for_edge_endpoint"
-EDGE_FILTERS = -m "not skip_for_edge_endpoint"
+CLOUD_FILTERS = -m "not run_only_for_edge_endpoint and not expensive"
+EDGE_FILTERS = -m "not skip_for_edge_endpoint and not expensive"
 
 # Record information about the slowest 25 tests (but don't show anything slower than 0.1 seconds)
 PROFILING_ARGS = \

pyproject.toml

@@ -84,6 +84,7 @@ trailing_comma_inline_array = true
 markers = [
     "skip_for_edge_endpoint",
     "run_only_for_edge_endpoint",
+    "expensive: marks tests as expensive (long-running; excluded from default test runs)",
 ]
 
 [build-system]

test/unit/test_priming_groups.py

@@ -1,257 +1,165 @@
-from unittest.mock import Mock, patch
+"""
+Tests for PrimingGroup and ML pipeline methods on ExperimentalApi.
+
+The create/get/delete priming group tests are marked @pytest.mark.expensive because they
+require training a detector (submit images, wait ~45 s) before a pipeline has a
+cached_vizlogic_key that can seed a PrimingGroup.  Run them explicitly with:
+
+    pytest -m expensive test/unit/test_priming_groups.py
+"""
+
+import time
 
 import pytest
 from groundlight import ExperimentalApi
 from groundlight.internalapi import NotFoundError
 from model import MLPipeline, PrimingGroup
 
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-MOCK_PIPELINE_DATA = {
-    "id": "pipe_abc123",
-    "pipeline_config": "never-review",
-    "cached_vizlogic_key": "mlb_testkey1234",
-    "is_active_pipeline": True,
-    "is_edge_pipeline": False,
-    "is_unclear_pipeline": False,
-    "is_oodd_pipeline": False,
-    "is_enabled": True,
-    "created_at": "2026-01-01T00:00:00Z",
-    "trained_at": None,
-}
-
-MOCK_PRIMING_GROUP_DATA = {
-    "id": "pgp_abc123",
-    "name": "door-detector-primer",
-    "is_global": False,
-    "canonical_query": "Is the door open?",
-    "active_pipeline_config": None,
-    "active_pipeline_base_mlbinary_key": "mlb_testkey1234",
-    "priming_group_specific_shadow_pipeline_configs": None,
-    "disable_shadow_pipelines": False,
-    "created_at": "2026-01-01T00:00:00Z",
-}
-
-
-def _mock_response(status_code=200, json_data=None):
-    resp = Mock()
-    resp.status_code = status_code
-    resp.json.return_value = json_data or {}
-    resp.raise_for_status.return_value = None
-    return resp
-
-
-def _mock_error_response(status_code):
-    resp = Mock()
-    resp.status_code = status_code
-    resp.raise_for_status.side_effect = Exception(f"HTTP {status_code}")
-    return resp
-
 
 # ---------------------------------------------------------------------------
 # list_detector_pipelines
 # ---------------------------------------------------------------------------
 
 
-def test_list_detector_pipelines_returns_pipelines():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_response(json_data={"results": [MOCK_PIPELINE_DATA], "count": 1})
-        pipelines = gl.list_detector_pipelines("det_abc123")
+def test_list_detector_pipelines_returns_list(gl_experimental: ExperimentalApi, detector):
+    """A freshly created detector has at least one pipeline."""
+    pipelines = gl_experimental.list_detector_pipelines(detector)
+    assert isinstance(pipelines, list)
+    assert len(pipelines) >= 1
+    assert all(isinstance(p, MLPipeline) for p in pipelines)
 
-    assert len(pipelines) == 1
-    assert isinstance(pipelines[0], MLPipeline)
-    assert pipelines[0].id == "pipe_abc123"
-    assert pipelines[0].is_active_pipeline is True
 
+def test_list_detector_pipelines_accepts_detector_id_string(gl_experimental: ExperimentalApi, detector):
+    """list_detector_pipelines should accept a raw ID string as well as a Detector object."""
+    by_obj = gl_experimental.list_detector_pipelines(detector)
+    by_id = gl_experimental.list_detector_pipelines(detector.id)
+    assert [p.id for p in by_obj] == [p.id for p in by_id]
 
-def test_list_detector_pipelines_accepts_detector_object():
-    gl = ExperimentalApi()
-    detector = Mock()
-    detector.id = "det_abc123"
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_response(json_data={"results": [MOCK_PIPELINE_DATA], "count": 1})
-        pipelines = gl.list_detector_pipelines(detector)
 
-    assert len(pipelines) == 1
-    assert mock_get.call_args[0][0].endswith("/v1/detectors/det_abc123/pipelines")
-
-
-def test_list_detector_pipelines_empty():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_response(json_data={"results": [], "count": 0})
-        pipelines = gl.list_detector_pipelines("det_abc123")
-
-    assert pipelines == []
-
-
-def test_list_detector_pipelines_404_raises_not_found():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_error_response(404)
-        with pytest.raises(NotFoundError, match="det_notexist"):
-            gl.list_detector_pipelines("det_notexist")
+def test_list_detector_pipelines_unknown_detector_raises(gl_experimental: ExperimentalApi):
+    with pytest.raises(NotFoundError):
+        gl_experimental.list_detector_pipelines("det_doesnotexist000000000000")
 
 
 # ---------------------------------------------------------------------------
 # list_priming_groups
 # ---------------------------------------------------------------------------
 
 
-def test_list_priming_groups_returns_groups():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_response(json_data={"results": [MOCK_PRIMING_GROUP_DATA], "count": 1})
-        groups = gl.list_priming_groups()
-
-    assert len(groups) == 1
-    assert isinstance(groups[0], PrimingGroup)
-    assert groups[0].id == "pgp_abc123"
-    assert groups[0].name == "door-detector-primer"
-
-
-def test_list_priming_groups_empty():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_response(json_data={"results": [], "count": 0})
-        groups = gl.list_priming_groups()
-
-    assert groups == []
-
-
-def test_list_priming_groups_hits_correct_url():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_response(json_data={"results": [], "count": 0})
-        gl.list_priming_groups()
-
-    called_url = mock_get.call_args[0][0]
-    assert called_url.endswith("/v1/priming-groups")
+def test_list_priming_groups_returns_list(gl_experimental: ExperimentalApi):
+    groups = gl_experimental.list_priming_groups()
+    assert isinstance(groups, list)
+    assert all(isinstance(g, PrimingGroup) for g in groups)
 
 
 # ---------------------------------------------------------------------------
-# create_priming_group
+# create / get / delete  (expensive — require a trained pipeline)
 # ---------------------------------------------------------------------------
 
 
-def test_create_priming_group_returns_group():
-    gl = ExperimentalApi()
-    with patch("requests.post") as mock_post:
-        mock_post.return_value = _mock_response(json_data=MOCK_PRIMING_GROUP_DATA)
-        pg = gl.create_priming_group(
-            name="door-detector-primer",
-            source_ml_pipeline_id="pipe_abc123",
-            canonical_query="Is the door open?",
-        )
-
-    assert isinstance(pg, PrimingGroup)
-    assert pg.id == "pgp_abc123"
-    assert pg.name == "door-detector-primer"
-    assert pg.canonical_query == "Is the door open?"
+def _wait_for_trained_pipeline(gl_experimental: ExperimentalApi, detector, timeout: int = 45) -> MLPipeline:
+    """
+    Submit the cat and dog test images, then poll until the active pipeline has a
+    cached_vizlogic_key (i.e. has been trained).  Raises TimeoutError if training
+    doesn't complete within `timeout` seconds.
+    """
+    gl_experimental.submit_image_query(detector, "test/assets/dog.jpeg", human_review="NEVER")
+    gl_experimental.submit_image_query(detector, "test/assets/cat.jpeg", human_review="NEVER")
 
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        pipelines = gl_experimental.list_detector_pipelines(detector)
+        for p in pipelines:
+            if p.is_active_pipeline and p.cached_vizlogic_key:
+                return p
+        time.sleep(5)
 
-def test_create_priming_group_sends_correct_payload():
-    gl = ExperimentalApi()
-    with patch("requests.post") as mock_post:
-        mock_post.return_value = _mock_response(json_data=MOCK_PRIMING_GROUP_DATA)
-        gl.create_priming_group(
-            name="door-detector-primer",
-            source_ml_pipeline_id="pipe_abc123",
-            canonical_query="Is the door open?",
-            disable_shadow_pipelines=True,
-        )
+    raise TimeoutError(f"Detector {detector.id} did not produce a trained pipeline within {timeout}s")
 
-    payload = mock_post.call_args[1]["json"]
-    assert payload["name"] == "door-detector-primer"
-    assert payload["source_ml_pipeline_id"] == "pipe_abc123"
-    assert payload["canonical_query"] == "Is the door open?"
-    assert payload["disable_shadow_pipelines"] is True
 
+@pytest.mark.expensive
+def test_create_priming_group(gl_experimental: ExperimentalApi, detector):
+    trained = _wait_for_trained_pipeline(gl_experimental, detector)
 
-def test_create_priming_group_omits_canonical_query_when_none():
-    gl = ExperimentalApi()
-    with patch("requests.post") as mock_post:
-        mock_post.return_value = _mock_response(json_data=MOCK_PRIMING_GROUP_DATA)
-        gl.create_priming_group(name="primer", source_ml_pipeline_id="pipe_abc123")
+    pg = gl_experimental.create_priming_group(
+        name=f"test-primer-{detector.id}",
+        source_ml_pipeline_id=trained.id,
+        canonical_query="Is there a dog?",
+    )
 
-    payload = mock_post.call_args[1]["json"]
-    assert "canonical_query" not in payload
+    assert isinstance(pg, PrimingGroup)
+    assert pg.id.startswith("pgp_")
+    assert pg.name == f"test-primer-{detector.id}"
+    assert pg.canonical_query == "Is there a dog?"
+    assert pg.is_global is False
+    assert pg.active_pipeline_base_mlbinary_key is not None
 
+    # cleanup
+    gl_experimental.delete_priming_group(pg.id)
 
-def test_create_priming_group_disable_shadow_pipelines_default_false():
-    gl = ExperimentalApi()
-    with patch("requests.post") as mock_post:
-        mock_post.return_value = _mock_response(json_data=MOCK_PRIMING_GROUP_DATA)
-        gl.create_priming_group(name="primer", source_ml_pipeline_id="pipe_abc123")
 
-    payload = mock_post.call_args[1]["json"]
-    assert payload["disable_shadow_pipelines"] is False
+@pytest.mark.expensive
+def test_create_priming_group_disable_shadow_pipelines(gl_experimental: ExperimentalApi, detector):
+    trained = _wait_for_trained_pipeline(gl_experimental, detector)
 
+    pg = gl_experimental.create_priming_group(
+        name=f"test-primer-noshadow-{detector.id}",
+        source_ml_pipeline_id=trained.id,
+        disable_shadow_pipelines=True,
+    )
 
-# ---------------------------------------------------------------------------
-# get_priming_group
-# ---------------------------------------------------------------------------
+    assert pg.disable_shadow_pipelines is True
 
+    gl_experimental.delete_priming_group(pg.id)
 
-def test_get_priming_group_returns_group():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_response(json_data=MOCK_PRIMING_GROUP_DATA)
-        pg = gl.get_priming_group("pgp_abc123")
 
-    assert isinstance(pg, PrimingGroup)
-    assert pg.id == "pgp_abc123"
-    assert pg.name == "door-detector-primer"
+@pytest.mark.expensive
+def test_get_priming_group(gl_experimental: ExperimentalApi, detector):
+    trained = _wait_for_trained_pipeline(gl_experimental, detector)
 
+    pg = gl_experimental.create_priming_group(
+        name=f"test-primer-get-{detector.id}",
+        source_ml_pipeline_id=trained.id,
+    )
 
-def test_get_priming_group_404_raises_not_found():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_error_response(404)
-        with pytest.raises(NotFoundError, match="pgp_notexist"):
-            gl.get_priming_group("pgp_notexist")
+    fetched = gl_experimental.get_priming_group(pg.id)
+    assert fetched.id == pg.id
+    assert fetched.name == pg.name
 
+    gl_experimental.delete_priming_group(pg.id)
 
-def test_get_priming_group_hits_correct_url():
-    gl = ExperimentalApi()
-    with patch("requests.get") as mock_get:
-        mock_get.return_value = _mock_response(json_data=MOCK_PRIMING_GROUP_DATA)
-        gl.get_priming_group("pgp_abc123")
 
-    called_url = mock_get.call_args[0][0]
-    assert called_url.endswith("/v1/priming-groups/pgp_abc123")
+@pytest.mark.expensive
+def test_get_priming_group_unknown_raises(gl_experimental: ExperimentalApi):
+    with pytest.raises(NotFoundError):
+        gl_experimental.get_priming_group("pgp_doesnotexist000000000000")
 
 
-# ---------------------------------------------------------------------------
-# delete_priming_group
-# ---------------------------------------------------------------------------
+@pytest.mark.expensive
+def test_delete_priming_group(gl_experimental: ExperimentalApi, detector):
+    trained = _wait_for_trained_pipeline(gl_experimental, detector)
 
+    pg = gl_experimental.create_priming_group(
+        name=f"test-primer-del-{detector.id}",
+        source_ml_pipeline_id=trained.id,
+    )
 
-def test_delete_priming_group_succeeds():
-    gl = ExperimentalApi()
-    with patch("requests.delete") as mock_delete:
-        mock_delete.return_value = _mock_response(status_code=204)
-        gl.delete_priming_group("pgp_abc123")  # should not raise
+    gl_experimental.delete_priming_group(pg.id)
 
-    mock_delete.assert_called_once()
+    with pytest.raises(NotFoundError):
+        gl_experimental.get_priming_group(pg.id)
 
 
-def test_delete_priming_group_404_raises_not_found():
-    gl = ExperimentalApi()
-    with patch("requests.delete") as mock_delete:
-        mock_delete.return_value = _mock_error_response(404)
-        with pytest.raises(NotFoundError, match="pgp_notexist"):
-            gl.delete_priming_group("pgp_notexist")
+@pytest.mark.expensive
+def test_created_priming_group_appears_in_list(gl_experimental: ExperimentalApi, detector):
+    trained = _wait_for_trained_pipeline(gl_experimental, detector)
 
+    pg = gl_experimental.create_priming_group(
+        name=f"test-primer-list-{detector.id}",
+        source_ml_pipeline_id=trained.id,
+    )
 
-def test_delete_priming_group_hits_correct_url():
-    gl = ExperimentalApi()
-    with patch("requests.delete") as mock_delete:
-        mock_delete.return_value = _mock_response(status_code=204)
-        gl.delete_priming_group("pgp_abc123")
+    groups = gl_experimental.list_priming_groups()
+    assert any(g.id == pg.id for g in groups)
 
-    called_url = mock_delete.call_args[0][0]
-    assert called_url.endswith("/v1/priming-groups/pgp_abc123")
+    gl_experimental.delete_priming_group(pg.id)