Skip to content

Commit 502996a

Browse files
author
Mateusz
committed
fix: improve alias:/auto: selector error messages and add startup warning for missing alias config
- Treat 'alias:' and 'auto:' as reserved selector namespaces in has_explicit_backend_selector() so they are never misparsed as concrete backend types (fixes 'Backend alias is not registered') - Add helpful runtime error hints in BackendRoutingService when alias:/auto: selectors fail: distinguish empty model_aliases (suggest --config) from no matching alias rule - Add startup warning warn_if_alias_references_without_rules() that fires when config settings reference alias:/auto: selectors but model_aliases is empty, listing the affected settings - Wire the startup warning into ApplicationBuilder validation cycle - Add regression tests for reserved namespace parsing, routing error hints (empty aliases, unmatched aliases, auto: namespace), and startup warning coverage - Add dev repro scripts demonstrating both fixes
1 parent e151a7c commit 502996a

10 files changed

Lines changed: 1561 additions & 902 deletions

‎dev/scripts/repro_runtime_alias_error.py‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
"""
2+
Repro script: Demonstrate improved runtime error messages for alias selectors.
3+
4+
This script proves that when an alias:/auto: selector fails to route,
5+
the error message now includes helpful hints about missing --config.
6+
"""
7+
8+
import sys
9+
from pathlib import Path
10+
11+
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
12+
13+
from unittest.mock import Mock
14+
from src.core.config.models.rewriting import ModelAliasRule
15+
from src.core.config.models.routing import RoutingConfig
16+
from src.core.services.backend_routing_service import BackendRoutingService
17+
from src.core.common.exceptions import RoutingError
18+
19+
20+
def test_runtime_error_with_empty_aliases():
21+
"""Test error message when alias: selector fails with empty model_aliases."""
22+
print("=" * 70)
23+
print("TEST 1: Runtime error with empty model_aliases")
24+
print("=" * 70)
25+
print()
26+
27+
# Mock config provider with empty aliases (simulates no --config)
28+
mock_provider = Mock()
29+
mock_provider._app_config = Mock(model_aliases=[])
30+
mock_provider.configs = {}
31+
mock_provider.get_backend_config.return_value = None
32+
mock_provider.iter_backend_names.return_value = []
33+
34+
service = BackendRoutingService(mock_provider, RoutingConfig())
35+
36+
try:
37+
service.resolve_model_only_backend("alias:oss-code-medium")
38+
except RoutingError as e:
39+
print("[PASS] RoutingError raised (expected)")
40+
print()
41+
print("Error message:")
42+
print("-" * 70)
43+
print(e.message)
44+
print("-" * 70)
45+
print()
46+
47+
# Verify the message contains helpful hints
48+
assert "No backend candidates discovered" in e.message
49+
assert "no `model_aliases` are loaded" in e.message
50+
assert "--config" in e.message
51+
print("[PASS] Error message contains helpful hint about --config")
52+
print()
53+
54+
55+
def test_runtime_error_with_unmatched_alias():
56+
"""Test error message when alias: selector fails with non-matching rule."""
57+
print("=" * 70)
58+
print("TEST 2: Runtime error with non-matching alias rule")
59+
print("=" * 70)
60+
print()
61+
62+
# Mock config provider with some aliases but none that match
63+
mock_provider = Mock()
64+
mock_provider._app_config = Mock(
65+
model_aliases=[
66+
ModelAliasRule(pattern=r"^alias:verifier$", replacement="openai:gpt-4"),
67+
]
68+
)
69+
mock_provider.configs = {}
70+
mock_provider.get_backend_config.return_value = None
71+
mock_provider.iter_backend_names.return_value = []
72+
73+
service = BackendRoutingService(mock_provider, RoutingConfig())
74+
75+
try:
76+
service.resolve_model_only_backend("alias:oss-code-medium")
77+
except RoutingError as e:
78+
print("[PASS] RoutingError raised (expected)")
79+
print()
80+
print("Error message:")
81+
print("-" * 70)
82+
print(e.message)
83+
print("-" * 70)
84+
print()
85+
86+
# Verify the message contains helpful hint about unmatched rule
87+
assert "No backend candidates discovered" in e.message
88+
assert "no configured alias matched 'alias:oss-code-medium'" in e.message
89+
print("[PASS] Error message explains that no alias rule matched")
90+
print()
91+
92+
93+
def test_runtime_error_with_auto_selector():
94+
"""Test error message for auto: selector."""
95+
print("=" * 70)
96+
print("TEST 3: Runtime error with auto: selector (no aliases)")
97+
print("=" * 70)
98+
print()
99+
100+
# Mock config provider with empty aliases
101+
mock_provider = Mock()
102+
mock_provider._app_config = Mock(model_aliases=[])
103+
mock_provider.configs = {}
104+
mock_provider.get_backend_config.return_value = None
105+
mock_provider.iter_backend_names.return_value = []
106+
107+
service = BackendRoutingService(mock_provider, RoutingConfig())
108+
109+
try:
110+
service.resolve_model_only_backend("auto:reasoning")
111+
except RoutingError as e:
112+
print("[PASS] RoutingError raised (expected)")
113+
print()
114+
print("Error message:")
115+
print("-" * 70)
116+
print(e.message)
117+
print("-" * 70)
118+
print()
119+
120+
# Verify the message contains helpful hints
121+
assert "No backend candidates discovered" in e.message
122+
assert "auto:" in e.message
123+
assert "--config" in e.message
124+
print("[PASS] Error message contains helpful hint about auto: selector")
125+
print()
126+
127+
128+
if __name__ == "__main__":
129+
print()
130+
print("DEMONSTRATING IMPROVED RUNTIME ERROR MESSAGES")
131+
print("This proves alias:/auto: selector failures now provide helpful hints")
132+
print()
133+
134+
test_runtime_error_with_empty_aliases()
135+
test_runtime_error_with_unmatched_alias()
136+
test_runtime_error_with_auto_selector()
137+
138+
print("=" * 70)
139+
print("ALL TESTS PASSED - Runtime error messages are now helpful!")
140+
print("=" * 70)
141+
print()
142+
print("Before fix: 'Unknown model alias:oss-code-medium'")
143+
print("After fix: 'Unknown model alias:oss-code-medium. No backend")
144+
print(" candidates discovered. The alias: selector namespace")
145+
print(" uses model alias rules, but no model_aliases are")
146+
print(" loaded. If you expected YAML aliases, verify the")
147+
print(" server was started with the intended --config file.'")
148+
print()

‎dev/scripts/repro_startup_warning.py‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
"""
2+
Repro script: Demonstrate startup warning for alias selectors with empty model_aliases.
3+
4+
This script proves that when the server starts with alias:/auto: selectors
5+
configured but no model_aliases loaded, a warning is emitted at startup.
6+
"""
7+
8+
import sys
9+
from pathlib import Path
10+
11+
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
12+
13+
import logging
14+
from unittest.mock import Mock
15+
import importlib
16+
17+
from src.core.config.app_config import AppConfig
18+
from src.core.config.models.rewriting import ModelAliasRule
19+
from src.core.config.semantic_validation import (
20+
warn_if_alias_references_without_rules,
21+
)
22+
23+
24+
def setup_logging():
25+
"""Setup logging to capture warnings."""
26+
logging.basicConfig(level=logging.WARNING, format='%(levelname)s: %(message)s')
27+
28+
29+
def test_startup_warning_with_quality_verifier_alias():
30+
"""Test warning when quality_verifier_model uses alias: but no aliases loaded."""
31+
print("=" * 70)
32+
print("TEST 1: Startup warning for quality_verifier_model='alias:verifier'")
33+
print(" with empty model_aliases (simulates missing --config)")
34+
print("=" * 70)
35+
print()
36+
37+
session = importlib.import_module("src.core.config.models.session")
38+
39+
# Config with alias selector but empty aliases (no --config scenario)
40+
config = AppConfig(
41+
session=session.SessionConfig(quality_verifier_model="alias:verifier"),
42+
model_aliases=[],
43+
)
44+
45+
print("Config settings:")
46+
print(" - quality_verifier_model: 'alias:verifier'")
47+
print(" - model_aliases: [] (empty)")
48+
print()
49+
50+
print("Startup warning emitted:")
51+
print("-" * 70)
52+
warn_if_alias_references_without_rules(config)
53+
print("-" * 70)
54+
print()
55+
print("[PASS] Warning was logged at startup")
56+
print()
57+
58+
59+
def test_startup_warning_with_static_route_alias():
60+
"""Test warning when static_route uses alias: but no aliases loaded."""
61+
print("=" * 70)
62+
print("TEST 2: Startup warning for static_route='alias:oss-code-medium'")
63+
print(" with empty model_aliases")
64+
print("=" * 70)
65+
print()
66+
67+
backends = importlib.import_module("src.core.config.models.backends")
68+
69+
config = AppConfig(
70+
backends=backends.BackendSettings(
71+
default_backend="openai",
72+
static_route="alias:oss-code-medium",
73+
),
74+
model_aliases=[],
75+
)
76+
77+
print("Config settings:")
78+
print(" - static_route: 'alias:oss-code-medium'")
79+
print(" - model_aliases: [] (empty)")
80+
print()
81+
82+
print("Startup warning emitted:")
83+
print("-" * 70)
84+
warn_if_alias_references_without_rules(config)
85+
print("-" * 70)
86+
print()
87+
print("[PASS] Warning mentions static_route setting")
88+
print()
89+
90+
91+
def test_no_warning_when_aliases_configured():
92+
"""Test that no warning when aliases are properly configured."""
93+
print("=" * 70)
94+
print("TEST 3: No warning when aliases are properly configured")
95+
print("=" * 70)
96+
print()
97+
98+
session = importlib.import_module("src.core.config.models.session")
99+
100+
config = AppConfig(
101+
session=session.SessionConfig(quality_verifier_model="alias:verifier"),
102+
model_aliases=[
103+
ModelAliasRule(pattern=r"^alias:verifier$", replacement="openai:gpt-4o"),
104+
],
105+
)
106+
107+
print("Config settings:")
108+
print(" - quality_verifier_model: 'alias:verifier'")
109+
print(" - model_aliases: [1 rule configured]")
110+
print()
111+
112+
print("Startup warning check:")
113+
print("-" * 70)
114+
# Capture log output
115+
import io
116+
log_capture = io.StringIO()
117+
handler = logging.StreamHandler(log_capture)
118+
handler.setLevel(logging.WARNING)
119+
logger = logging.getLogger("src.core.config.semantic_validation")
120+
original_handlers = list(logger.handlers)
121+
logger.handlers = [handler]
122+
123+
warn_if_alias_references_without_rules(config)
124+
125+
output = log_capture.getvalue()
126+
logger.handlers = original_handlers
127+
128+
if "alias:/auto: selectors" in output:
129+
print("ERROR: Warning was emitted but shouldn't have been!")
130+
else:
131+
print("(No warning - correct behavior)")
132+
print("-" * 70)
133+
print()
134+
print("[PASS] No warning when aliases are properly configured")
135+
print()
136+
137+
138+
if __name__ == "__main__":
139+
print()
140+
print("DEMONSTRATING STARTUP WARNING FOR MISSING ALIAS CONFIG")
141+
print("This proves the server warns at startup when alias:/auto:")
142+
print("selectors are configured but model_aliases is empty")
143+
print()
144+
145+
setup_logging()
146+
test_startup_warning_with_quality_verifier_alias()
147+
test_startup_warning_with_static_route_alias()
148+
test_no_warning_when_aliases_configured()
149+
150+
print("=" * 70)
151+
print("ALL TESTS PASSED - Startup warnings are working correctly!")
152+
print("=" * 70)
153+
print()
154+
print("When you start the server without --config but have alias:")
155+
print("selectors configured, you'll now see:")
156+
print()
157+
print("WARNING: The following settings use alias:/auto: selectors,")
158+
print(" but model_aliases is empty. If you expected YAML")
159+
print(" aliases, restart with the intended --config file.")
160+
print(" Affected settings: quality_verifier_model='alias:verifier'")
161+
print()

‎src/core/app/application_builder.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,12 +330,14 @@ async def build(self, config: AppConfig) -> FastAPI:
330330
validate_extracted_backend_references,
331331
validate_model_aliases,
332332
validate_static_route,
333+
warn_if_alias_references_without_rules,
333334
)
334335

335336
validate_static_route(config)
336337
validate_extracted_backend_references(config)
337338
validate_constrained_backend_instances(config)
338339
validate_model_aliases(config)
340+
warn_if_alias_references_without_rules(config)
339341

340342
# Replace DI-registered AppConfig and IConfig with runtime config instance
341343
# This ensures validation services see the same config that the builder was given

‎src/core/config/semantic_validation.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -656,6 +656,66 @@ def _collect_leaf_nodes(node):
656656
)
657657

658658

659+
def _is_alias_selector(value: str | None) -> bool:
660+
"""Check if a selector string uses alias: or auto: namespace."""
661+
if not value or ":" not in value:
662+
return False
663+
namespace, _, _ = value.partition(":")
664+
return namespace.strip().lower() in {"alias", "auto"}
665+
666+
667+
def warn_if_alias_references_without_rules(config: AppConfig) -> None:
668+
"""Warn at startup when config references alias:/auto: selectors but model_aliases is empty.
669+
670+
This is a fail-open warning (no exception raised) intended to surface the
671+
common misconfiguration where the server starts without `--config` and
672+
therefore has no alias rules, while some settings still reference alias
673+
selectors.
674+
"""
675+
alias_rules = getattr(config, "model_aliases", None)
676+
if isinstance(alias_rules, list) and alias_rules:
677+
return
678+
679+
hints: list[str] = []
680+
681+
session_cfg = getattr(config, "session", None)
682+
if session_cfg is not None:
683+
qv_model = getattr(session_cfg, "quality_verifier_model", None)
684+
if _is_alias_selector(qv_model):
685+
hints.append(f"session.quality_verifier_model='{qv_model}'")
686+
687+
backends_cfg = getattr(config, "backends", None)
688+
if backends_cfg is not None:
689+
static_route = getattr(backends_cfg, "static_route", None)
690+
if _is_alias_selector(static_route):
691+
hints.append(f"backends.static_route='{static_route}'")
692+
693+
aux_cfg = getattr(config, "auxiliary_routing", None)
694+
if aux_cfg is not None:
695+
aux_model = getattr(aux_cfg, "model", None)
696+
if _is_alias_selector(aux_model):
697+
hints.append(f"auxiliary_routing.model='{aux_model}'")
698+
699+
replacement_rules = getattr(config, "replacement_rules", None)
700+
if isinstance(replacement_rules, list):
701+
for idx, rule in enumerate(replacement_rules):
702+
to_selector = getattr(rule, "to_backend_model", None) or getattr(
703+
rule, "replacement", None
704+
)
705+
if _is_alias_selector(to_selector):
706+
hints.append(f"replacement_rules[{idx}].to='{to_selector}'")
707+
708+
if not hints:
709+
return
710+
711+
logger.warning(
712+
"The following settings use alias:/auto: selectors, but model_aliases "
713+
"is empty. If you expected YAML aliases, restart with the intended "
714+
"--config file. Affected settings: %s",
715+
"; ".join(hints),
716+
)
717+
718+
659719
def validate_model_aliases(config: AppConfig) -> None:
660720
"""Validate model alias patterns and replacement routing strings at startup.
661721

0 commit comments

Comments
 (0)