fix(computer_use): honor custom vision routing

This commit is contained in:
helix4u 2026-06-03 21:03:31 -06:00 committed by Teknium
parent ffe665277c
commit 591e6fb8f4
6 changed files with 207 additions and 7 deletions

View file

@ -28,6 +28,10 @@ Behaviour (mirrors ``vision_analyze`` for consistency)
``provider``, ``model``, or ``base_url`` non-empty / not ``"auto"``),
the screenshot is routed through the aux vision pipeline. Users who
pay for a dedicated vision model usually want it used.
* Otherwise, if the user explicitly declared the active model vision-capable
via ``model.supports_vision`` / provider model config, return ``False``.
This is the escape hatch for custom/local OpenAI-compatible VLM routes that
are absent from models.dev and provider allowlists.
* Otherwise, if the active main model+provider can carry an image inside
a tool-result message AND the model reports ``supports_vision=True``
in models.dev metadata, return ``False`` (use the multimodal path).
@ -76,10 +80,52 @@ def _explicit_aux_vision_override(cfg: Optional[Dict[str, Any]]) -> bool:
return True
def _lookup_supports_vision(provider: str, model: str) -> Optional[bool]:
"""Return models.dev ``supports_vision`` for *(provider, model)* or None."""
def _lookup_user_declared_supports_vision(
provider: str,
model: str,
cfg: Optional[Dict[str, Any]],
) -> Optional[bool]:
"""Return config-declared ``supports_vision`` for the active route."""
try:
from agent.image_routing import _supports_vision_override
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"computer_use vision_routing: config override lookup import failed: %s",
exc,
)
return None
try:
return _supports_vision_override(cfg, provider, model)
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"computer_use vision_routing: config override lookup failed: %s",
exc,
)
return None
def _lookup_supports_vision(
provider: str,
model: str,
cfg: Optional[Dict[str, Any]] = None,
) -> Optional[bool]:
"""Return config/models.dev ``supports_vision`` for *(provider, model)*."""
if not provider or not model:
return None
try:
from agent.image_routing import _lookup_supports_vision as _lookup_image_supports
except Exception:
_lookup_image_supports = None
if _lookup_image_supports is not None:
try:
return _lookup_image_supports(provider, model, cfg)
except Exception as exc: # pragma: no cover - defensive
logger.debug(
"computer_use vision_routing: image-routing caps lookup failed "
"for %s:%s%s",
provider, model, exc,
)
return None
try:
from agent.models_dev import get_model_capabilities
caps = get_model_capabilities(provider, model)
@ -137,11 +183,17 @@ def should_route_capture_to_aux_vision(
if _explicit_aux_vision_override(cfg):
return True
user_declared = _lookup_user_declared_supports_vision(provider, model, cfg)
if user_declared is True:
return False
if user_declared is False:
return True
accepts_tool_image = _provider_accepts_multimodal_tool_result(provider, model)
if accepts_tool_image is None or accepts_tool_image is False:
return True
supports_vision = _lookup_supports_vision(provider, model)
supports_vision = _lookup_supports_vision(provider, model, cfg)
if supports_vision is True:
return False
return True