address PR review: json schema structured outputs, tighten vllm range

- Switch from choice-based structured outputs (121 cartesian product
  entries) to JSON schema constraint via StructuredOutputsParams(json=...).
  This scales to multi-criteria evaluation without combinatorial explosion.
- Tighten vllm version range from >=0.17.0,<1.0.0 to >=0.17.0,<0.19.0
  (tested with 0.18.1 on cluster).
- Update tests to validate JSON schema structure.

Includes-AI-Code: true
Made-with: Cursor

Author: ErlisLushtaku

judgearena/evaluate.py

@@ -56,12 +56,21 @@ def get_regexp_match(self, s: str, regex: str, group_index: int = 1):
 _PAIR_SCORE_MAX = 10
 
 
-def build_pair_score_output_choices() -> list[str]:
-    return [
-        f"score_A: {a}\nscore_B: {b}"
-        for a in range(_PAIR_SCORE_MIN, _PAIR_SCORE_MAX + 1)
-        for b in range(_PAIR_SCORE_MIN, _PAIR_SCORE_MAX + 1)
-    ]
+def build_pair_score_json_schema() -> dict:
+    score_field = {
+        "type": "integer",
+        "minimum": _PAIR_SCORE_MIN,
+        "maximum": _PAIR_SCORE_MAX,
+    }
+    return {
+        "type": "object",
+        "properties": {
+            "score_A": score_field,
+            "score_B": score_field,
+        },
+        "required": ["score_A", "score_B"],
+        "additionalProperties": False,
+    }
 
 
 _COMPLETION_LABEL_SINGLE = "Answer"

judgearena/generate_and_evaluate.py

@@ -13,7 +13,7 @@
 import pandas as pd
 
 from judgearena.evaluate import (
-    build_pair_score_output_choices,
+    build_pair_score_json_schema,
     judge_and_parse_prefs,
     resolve_judge_prompts,
 )
@@ -407,8 +407,8 @@ def main(args: CliArgs):
 
     judge_model_kwargs = dict(args.engine_kwargs)
     if not args.provide_explanation and args.judge_model.split("/")[0] == "VLLM":
-        judge_model_kwargs["structured_outputs_choice"] = (
-            build_pair_score_output_choices()
+        judge_model_kwargs["structured_outputs_json"] = (
+            build_pair_score_json_schema()
         )
 
     judge_chat_model = make_model(

judgearena/utils.py

@@ -236,10 +236,10 @@ def __init__(
             "temperature": float(vllm_kwargs.pop("temperature", 0.6)),
             "top_p": float(vllm_kwargs.pop("top_p", 0.95)),
         }
-        structured_outputs_choice = vllm_kwargs.pop("structured_outputs_choice", None)
-        if structured_outputs_choice is not None:
+        structured_outputs_json = vllm_kwargs.pop("structured_outputs_json", None)
+        if structured_outputs_json is not None:
             self._sampling_params_kwargs["structured_outputs"] = (
-                StructuredOutputsParams(choice=structured_outputs_choice)
+                StructuredOutputsParams(json=structured_outputs_json)
             )
         self.sampling_params = SamplingParams(**self._sampling_params_kwargs)
 

pyproject.toml

@@ -82,5 +82,6 @@ indent-style = "space"
 
 [project.optional-dependencies]
 # vLLM on PyPI pins transformers<5; optional extra matches that so `uv lock` can resolve.
-vllm = ["vllm>=0.17.0,<1.0.0", "transformers>=4.56.0,<5.0.0"]
+# Tested with vllm 0.18.1; StructuredOutputsParams(json=...) requires >= 0.17.
+vllm = ["vllm>=0.17.0,<0.19.0", "transformers>=4.56.0,<5.0.0"]
 llamacpp = ["llama-cpp-python>=0.3.0"]

tests/test_local_completion_loading.py

@@ -6,13 +6,16 @@
 from judgearena.generate_and_evaluate import main as main_generate_and_eval
 
 
-def test_build_pair_score_output_choices_covers_all_integer_pairs():
-    choices = evaluate.build_pair_score_output_choices()
-
-    assert len(choices) == 121
-    assert len(set(choices)) == 121
-    assert "score_A: 0\nscore_B: 0" in choices
-    assert "score_A: 10\nscore_B: 10" in choices
+def test_build_pair_score_json_schema_covers_valid_range():
+    schema = evaluate.build_pair_score_json_schema()
+
+    assert schema["type"] == "object"
+    assert set(schema["required"]) == {"score_A", "score_B"}
+    for key in ("score_A", "score_B"):
+        assert schema["properties"][key]["type"] == "integer"
+        assert schema["properties"][key]["minimum"] == 0
+        assert schema["properties"][key]["maximum"] == 10
+    assert schema["additionalProperties"] is False
 
 
 def test_main_aligns_local_reference_by_instruction_index(tmp_path, monkeypatch):