Merge pull request #30 from sacredvoid/fix/parse-results-bool-filter

fix(eval): exclude booleans from parsed benchmark metrics

Author: sacredvoid

src/alignrl/eval.py

@@ -28,7 +28,10 @@ def parse_results(raw: dict[str, Any], model_name: str, stage: str) -> EvalResul
     """Parse lm-evaluation-harness output into EvalResult."""
     benchmarks: dict[str, dict[str, float]] = {}
     for task_name, metrics in raw.get("results", {}).items():
-        benchmarks[task_name] = {k: v for k, v in metrics.items() if isinstance(v, (int, float))}
+        benchmarks[task_name] = {
+            k: v for k, v in metrics.items()
+            if isinstance(v, (int, float)) and not isinstance(v, bool)
+        }
     return EvalResult(model_name=model_name, stage=stage, benchmarks=benchmarks)
 
 

tests/test_eval.py

@@ -171,3 +171,18 @@ def test_filters_non_numeric(self) -> None:
     def test_no_results_key(self) -> None:
         result = parse_results({}, model_name="test", stage="base")
         assert result.benchmarks == {}
+
+    def test_filters_booleans(self) -> None:
+        raw = {
+            "results": {
+                "gsm8k": {
+                    "exact_match": 0.5,
+                    "has_config": True,
+                    "is_valid": False,
+                }
+            }
+        }
+        result = parse_results(raw, model_name="test", stage="base")
+        assert "exact_match" in result.benchmarks["gsm8k"]
+        assert "has_config" not in result.benchmarks["gsm8k"]
+        assert "is_valid" not in result.benchmarks["gsm8k"]