fix(computer_use): honor custom vision routing
This commit is contained in:
parent
ffe665277c
commit
591e6fb8f4
6 changed files with 207 additions and 7 deletions
|
|
@ -28,6 +28,10 @@ Behaviour (mirrors ``vision_analyze`` for consistency)
|
|||
``provider``, ``model``, or ``base_url`` non-empty / not ``"auto"``),
|
||||
the screenshot is routed through the aux vision pipeline. Users who
|
||||
pay for a dedicated vision model usually want it used.
|
||||
* Otherwise, if the user explicitly declared the active model vision-capable
|
||||
via ``model.supports_vision`` / provider model config, return ``False``.
|
||||
This is the escape hatch for custom/local OpenAI-compatible VLM routes that
|
||||
are absent from models.dev and provider allowlists.
|
||||
* Otherwise, if the active main model+provider can carry an image inside
|
||||
a tool-result message AND the model reports ``supports_vision=True``
|
||||
in models.dev metadata, return ``False`` (use the multimodal path).
|
||||
|
|
@ -76,10 +80,52 @@ def _explicit_aux_vision_override(cfg: Optional[Dict[str, Any]]) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _lookup_supports_vision(provider: str, model: str) -> Optional[bool]:
|
||||
"""Return models.dev ``supports_vision`` for *(provider, model)* or None."""
|
||||
def _lookup_user_declared_supports_vision(
|
||||
provider: str,
|
||||
model: str,
|
||||
cfg: Optional[Dict[str, Any]],
|
||||
) -> Optional[bool]:
|
||||
"""Return config-declared ``supports_vision`` for the active route."""
|
||||
try:
|
||||
from agent.image_routing import _supports_vision_override
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug(
|
||||
"computer_use vision_routing: config override lookup import failed: %s",
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
try:
|
||||
return _supports_vision_override(cfg, provider, model)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug(
|
||||
"computer_use vision_routing: config override lookup failed: %s",
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _lookup_supports_vision(
|
||||
provider: str,
|
||||
model: str,
|
||||
cfg: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[bool]:
|
||||
"""Return config/models.dev ``supports_vision`` for *(provider, model)*."""
|
||||
if not provider or not model:
|
||||
return None
|
||||
try:
|
||||
from agent.image_routing import _lookup_supports_vision as _lookup_image_supports
|
||||
except Exception:
|
||||
_lookup_image_supports = None
|
||||
if _lookup_image_supports is not None:
|
||||
try:
|
||||
return _lookup_image_supports(provider, model, cfg)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
logger.debug(
|
||||
"computer_use vision_routing: image-routing caps lookup failed "
|
||||
"for %s:%s — %s",
|
||||
provider, model, exc,
|
||||
)
|
||||
return None
|
||||
try:
|
||||
from agent.models_dev import get_model_capabilities
|
||||
caps = get_model_capabilities(provider, model)
|
||||
|
|
@ -137,11 +183,17 @@ def should_route_capture_to_aux_vision(
|
|||
if _explicit_aux_vision_override(cfg):
|
||||
return True
|
||||
|
||||
user_declared = _lookup_user_declared_supports_vision(provider, model, cfg)
|
||||
if user_declared is True:
|
||||
return False
|
||||
if user_declared is False:
|
||||
return True
|
||||
|
||||
accepts_tool_image = _provider_accepts_multimodal_tool_result(provider, model)
|
||||
if accepts_tool_image is None or accepts_tool_image is False:
|
||||
return True
|
||||
|
||||
supports_vision = _lookup_supports_vision(provider, model)
|
||||
supports_vision = _lookup_supports_vision(provider, model, cfg)
|
||||
if supports_vision is True:
|
||||
return False
|
||||
return True
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue