fix(model_switch): route typed configured models off openai-codex (#45006)
A typed `/model <name>` where `<name>` is declared under `providers.<slug>` or `custom_providers` — but typed while the current provider is a soft-accepting one (e.g. `openai-codex`) — stayed on the current provider and was swallowed as an unknown hidden Codex model, instead of routing to the provider that actually declares it. Add configured-provider exact-match detection (`_configured_provider_matches`) and a new Step d.5 in `switch_model`: if the typed model is declared in user/custom provider config, route to that provider BEFORE `detect_provider_for_model()` guesses from static catalogs and BEFORE the common-path validation lets a soft-accepting current provider swallow the name. - Matching is exact (case-insensitive) against explicitly-declared model collections only (`models`, `model`, `default_model`) — never fuzzy/family. - Same-provider declarer → keep current provider (canonicalize the id). - Multiple declarers → fail clearly and ask for `--provider <slug>`. - Single declarer → route there; for `providers.<slug>` user providers, set `explicit_provider` so the credential block resolves base_url/key from config. - Step e (`detect_provider_for_model`) is gated off when `config_routed`. The deliberately-supported openai-codex / xai-oauth hidden-model soft-accept (#16172 / #19729) is left untouched: when nothing in config matches, detection is a no-op. Salvaged from #45442 by harjothkhara (authorship preserved). Tests: tests/hermes_cli/test_model_switch_configured_provider_routing.py (7 tests). Full model_switch suite: 214 passed. Fixes #45006
This commit is contained in:
harjothkhara
parent
2a58fee1a1
commit
791c992b55
2 changed files with 445 additions and 0 deletions
|
|
@ -0,0 +1,310 @@
|
|||
"""Regression tests for #45006: typed `/model <name>` resolution must route a
|
||||
model declared in user/custom provider config to that provider instead of
|
||||
leaving it on the current provider and soft-accepting it.
|
||||
|
||||
Repro: with the current provider set to ``openai-codex``, typing
|
||||
``/model qwen3.5-4b`` (a model the user declares under ``providers.<slug>`` or
|
||||
``custom_providers``) showed ``Provider: OpenAI Codex`` — because typed
|
||||
detection only consulted static catalogs / OpenRouter, never the user's
|
||||
configured provider model lists, so the name stayed on Codex and was
|
||||
soft-accepted as an unknown hidden Codex model.
|
||||
|
||||
The fix adds an exact-match configured-provider detection step in
|
||||
``switch_model`` that runs before ``detect_provider_for_model`` and before
|
||||
common-path validation. These tests pin its precedence rules and prove the
|
||||
deliberately-supported Codex hidden-model soft-accept (#16172 / #19729) is left
|
||||
intact when nothing in config matches.
|
||||
|
||||
Hermetic: the model-resolution chain is fully mocked (no network), mirroring
|
||||
``tests/hermes_cli/test_user_providers_model_switch.py``.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
from hermes_cli.model_switch import switch_model
|
||||
|
||||
_ACCEPTED = {"accepted": True, "persist": True, "recognized": True, "message": None}
|
||||
_REJECTED = {"accepted": False, "persist": False, "recognized": False, "message": "not found"}
|
||||
# What validate_requested_model returns for an unknown id on openai-codex: it
|
||||
# soft-accepts with a "may be a hidden model" note (#16172 / #19729).
|
||||
_CODEX_SOFT_ACCEPT = {
|
||||
"accepted": True,
|
||||
"persist": True,
|
||||
"recognized": False,
|
||||
"message": (
|
||||
"Note: `gpt-5.9-codex-hidden` was not found in the OpenAI Codex model "
|
||||
"listing. It may still work if your account has access to a newer or "
|
||||
"hidden model ID."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _run_switch(
|
||||
*,
|
||||
raw_input,
|
||||
current_provider,
|
||||
user_providers=None,
|
||||
custom_providers=None,
|
||||
validation=_ACCEPTED,
|
||||
current_model="old-model",
|
||||
current_base_url="",
|
||||
):
|
||||
"""Drive ``switch_model`` with the resolution chain mocked out.
|
||||
|
||||
Every external lookup that would otherwise hit catalogs/network is patched:
|
||||
alias resolution, aggregator catalog, ``detect_provider_for_model`` (so step
|
||||
e is a no-op and cannot accidentally reroute), validation, credential
|
||||
resolution, normalization, and model metadata. This isolates the new
|
||||
configured-provider detection step.
|
||||
"""
|
||||
with patch("hermes_cli.model_switch.resolve_alias", return_value=None), \
|
||||
patch("hermes_cli.model_switch.list_provider_models", return_value=[]), \
|
||||
patch("hermes_cli.model_switch.normalize_model_for_provider", side_effect=lambda model, provider: model), \
|
||||
patch("hermes_cli.models.validate_requested_model", return_value=validation), \
|
||||
patch("hermes_cli.models.detect_provider_for_model", return_value=None), \
|
||||
patch("hermes_cli.model_switch.get_model_info", return_value=None), \
|
||||
patch("hermes_cli.model_switch.get_model_capabilities", return_value=None), \
|
||||
patch(
|
||||
"hermes_cli.runtime_provider.resolve_runtime_provider",
|
||||
return_value={
|
||||
"api_key": "***",
|
||||
"base_url": current_base_url or "http://resolved/v1",
|
||||
"api_mode": "",
|
||||
},
|
||||
):
|
||||
return switch_model(
|
||||
raw_input=raw_input,
|
||||
current_provider=current_provider,
|
||||
current_model=current_model,
|
||||
current_base_url=current_base_url,
|
||||
user_providers=user_providers or {},
|
||||
custom_providers=custom_providers or [],
|
||||
)
|
||||
|
||||
|
||||
def test_typed_configured_model_routes_away_from_openai_codex():
|
||||
"""The core repro: a model declared under ``providers.<slug>`` typed while
|
||||
on ``openai-codex`` routes to the configured provider, not Codex."""
|
||||
user_providers = {
|
||||
"local-ollama": {
|
||||
"name": "Local Ollama",
|
||||
"base_url": "http://localhost:11434/v1",
|
||||
"models": ["qwen3.5-4b", "kimi-k2.5"],
|
||||
}
|
||||
}
|
||||
result = _run_switch(
|
||||
raw_input="qwen3.5-4b",
|
||||
current_provider="openai-codex",
|
||||
current_model="gpt-5.4",
|
||||
user_providers=user_providers,
|
||||
)
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "local-ollama"
|
||||
assert result.new_model == "qwen3.5-4b"
|
||||
|
||||
|
||||
def test_typed_configured_model_routes_to_custom_provider():
|
||||
"""``custom_providers`` entries route to their ``custom:<name>`` slug."""
|
||||
custom_providers = [
|
||||
{
|
||||
"name": "mylocal",
|
||||
"base_url": "http://localhost:1234/v1",
|
||||
"model": "qwen3.5-4b",
|
||||
"models": {"qwen3.5-4b": {}},
|
||||
}
|
||||
]
|
||||
result = _run_switch(
|
||||
raw_input="qwen3.5-4b",
|
||||
current_provider="openai-codex",
|
||||
current_model="gpt-5.4",
|
||||
custom_providers=custom_providers,
|
||||
)
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "custom:mylocal"
|
||||
assert result.new_model == "qwen3.5-4b"
|
||||
|
||||
|
||||
def test_current_provider_declaring_model_is_not_rerouted():
|
||||
"""Precedence rule 4: if the current provider declares the model, keep it —
|
||||
even when another configured provider also declares the same id (so this
|
||||
must NOT trip the ambiguity guard)."""
|
||||
user_providers = {
|
||||
"local-ollama": {
|
||||
"name": "Local Ollama",
|
||||
"base_url": "http://localhost:11434/v1",
|
||||
"models": ["qwen3.5-4b"],
|
||||
},
|
||||
"other-relay": {
|
||||
"name": "Other Relay",
|
||||
"base_url": "http://other/v1",
|
||||
"models": ["qwen3.5-4b"],
|
||||
},
|
||||
}
|
||||
result = _run_switch(
|
||||
raw_input="qwen3.5-4b",
|
||||
current_provider="local-ollama",
|
||||
current_model="kimi-k2.5",
|
||||
current_base_url="http://localhost:11434/v1",
|
||||
user_providers=user_providers,
|
||||
)
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "local-ollama"
|
||||
|
||||
|
||||
def test_ambiguous_configured_model_fails_with_provider_hint():
|
||||
"""Precedence rule 6: when two non-current providers declare the same id and
|
||||
neither is current, fail clearly and point at ``--provider`` — never
|
||||
silently pick the first match."""
|
||||
user_providers = {
|
||||
"relay-a": {
|
||||
"name": "Relay A",
|
||||
"base_url": "http://a/v1",
|
||||
"models": ["qwen3.5-4b"],
|
||||
},
|
||||
"relay-b": {
|
||||
"name": "Relay B",
|
||||
"base_url": "http://b/v1",
|
||||
"models": ["qwen3.5-4b"],
|
||||
},
|
||||
}
|
||||
result = _run_switch(
|
||||
raw_input="qwen3.5-4b",
|
||||
current_provider="openai-codex",
|
||||
current_model="gpt-5.4",
|
||||
user_providers=user_providers,
|
||||
)
|
||||
assert result.success is False
|
||||
assert "--provider" in result.error_message
|
||||
assert "relay-a" in result.error_message
|
||||
assert "relay-b" in result.error_message
|
||||
|
||||
|
||||
def test_configured_model_absent_from_live_models_accepted_after_reroute():
|
||||
"""End-to-end synergy: after rerouting to the configured provider, a live
|
||||
``/v1/models`` probe that does NOT list the model is still accepted via the
|
||||
existing user-config override — proving the reroute lands on the right
|
||||
provider for that override to match."""
|
||||
user_providers = {
|
||||
"local-ollama": {
|
||||
"name": "Local Ollama",
|
||||
"base_url": "http://localhost:11434/v1",
|
||||
"models": {"qwen3.5-4b": {"context_length": 32768}},
|
||||
}
|
||||
}
|
||||
result = _run_switch(
|
||||
raw_input="qwen3.5-4b",
|
||||
current_provider="openai-codex",
|
||||
current_model="gpt-5.4",
|
||||
user_providers=user_providers,
|
||||
validation=_REJECTED,
|
||||
)
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "local-ollama"
|
||||
assert result.new_model == "qwen3.5-4b"
|
||||
|
||||
|
||||
def test_no_configured_match_leaves_current_provider_for_soft_accept():
|
||||
"""The Codex hidden-model soft-accept (#16172 / #19729) is untouched: an
|
||||
unknown id with no config match stays on the current provider and is
|
||||
soft-accepted exactly as before."""
|
||||
result = _run_switch(
|
||||
raw_input="gpt-5.9-codex-hidden",
|
||||
current_provider="openai-codex",
|
||||
current_model="gpt-5.4",
|
||||
# Config is present but declares an unrelated model — detection is a no-op.
|
||||
user_providers={
|
||||
"local-ollama": {
|
||||
"base_url": "http://localhost:11434/v1",
|
||||
"models": ["qwen3.5-4b"],
|
||||
}
|
||||
},
|
||||
validation=_CODEX_SOFT_ACCEPT,
|
||||
)
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "openai-codex"
|
||||
assert result.new_model == "gpt-5.9-codex-hidden"
|
||||
|
||||
|
||||
def test_configured_match_is_case_insensitive_and_returns_canonical_spelling():
|
||||
"""Matching is case-insensitive but the configured spelling wins, so the
|
||||
downstream validation/override path sees the canonical id."""
|
||||
user_providers = {
|
||||
"local-ollama": {
|
||||
"base_url": "http://localhost:11434/v1",
|
||||
"models": ["Qwen3.5-4B"],
|
||||
}
|
||||
}
|
||||
result = _run_switch(
|
||||
raw_input="qwen3.5-4b",
|
||||
current_provider="openai-codex",
|
||||
current_model="gpt-5.4",
|
||||
user_providers=user_providers,
|
||||
)
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "local-ollama"
|
||||
assert result.new_model == "Qwen3.5-4B"
|
||||
|
||||
|
||||
def test_default_model_only_declaration_routes():
|
||||
"""A model declared ONLY via `default_model` (not in `models`) still routes
|
||||
to that configured provider (#45006 — default_model is a declaring field)."""
|
||||
user_providers = {
|
||||
"local-ollama": {
|
||||
"name": "Local Ollama",
|
||||
"base_url": "http://localhost:11434/v1",
|
||||
"default_model": "qwen3.5-4b",
|
||||
}
|
||||
}
|
||||
result = _run_switch(
|
||||
raw_input="qwen3.5-4b",
|
||||
current_provider="openai-codex",
|
||||
current_model="gpt-5.4",
|
||||
user_providers=user_providers,
|
||||
)
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "local-ollama"
|
||||
assert result.new_model == "qwen3.5-4b"
|
||||
|
||||
|
||||
def test_malformed_provider_config_does_not_raise():
|
||||
"""Garbage shapes in provider config must not crash detection — they're
|
||||
skipped and the typed name falls through to the soft-accept no-op."""
|
||||
user_providers = {
|
||||
"bad1": "not-a-dict", # non-dict cfg
|
||||
"bad2": {"models": 12345}, # models as int
|
||||
"bad3": {"models": [None, 7, {"noname": "x"}]}, # junk list items
|
||||
"bad4": {"model": {"k": object()}}, # dict with non-target keys
|
||||
}
|
||||
custom_providers = [
|
||||
"not-a-dict", # non-dict entry
|
||||
{"name": ""}, # empty name
|
||||
{"models": ["unrelated-model"]}, # no name key
|
||||
]
|
||||
result = _run_switch(
|
||||
raw_input="gpt-5.9-codex-hidden",
|
||||
current_provider="openai-codex",
|
||||
current_model="gpt-5.4",
|
||||
user_providers=user_providers,
|
||||
custom_providers=custom_providers,
|
||||
validation=_CODEX_SOFT_ACCEPT,
|
||||
)
|
||||
# No match anywhere -> stays on codex, soft-accepted, no exception.
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "openai-codex"
|
||||
|
||||
|
||||
def test_xai_oauth_soft_accept_preserved_when_no_match():
|
||||
"""The xai-oauth hidden-model soft-accept (sibling of openai-codex) is also
|
||||
a no-op when config declares no matching model."""
|
||||
user_providers = {
|
||||
"local-ollama": {"base_url": "http://x/v1", "models": ["some-other-model"]},
|
||||
}
|
||||
result = _run_switch(
|
||||
raw_input="grok-hidden-preview",
|
||||
current_provider="xai-oauth",
|
||||
current_model="grok-4",
|
||||
user_providers=user_providers,
|
||||
validation=_CODEX_SOFT_ACCEPT,
|
||||
)
|
||||
assert result.success is True, result.error_message
|
||||
assert result.target_provider == "xai-oauth"
|
||||
Loading…
Add table
Add a link
Reference in a new issue