Merge remote-tracking branch 'origin/main' into hermes/hermes-6b48295e
This commit is contained in:
commit
2ecb4e62bb
239 changed files with 18356 additions and 2494 deletions
96
tests/agent/test_anthropic_output_field_leak.py
Normal file
96
tests/agent/test_anthropic_output_field_leak.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
"""Regression: output-only SDK fields must not leak into Anthropic request input.
|
||||
|
||||
Reproduces HTTP 400 `messages.N.content.M.text.parsed_output: Extra inputs are
|
||||
not permitted`. Anthropic SDK response blocks carry output-only attributes
|
||||
(text blocks: `parsed_output`, `citations=None`; tool_use blocks: `caller`)
|
||||
that the Messages *input* schema forbids. normalize_response captured blocks
|
||||
verbatim via _to_plain_data and replayed them as input → 400.
|
||||
|
||||
Fix: whitelist input-permitted fields per block type at three points —
|
||||
normalize_response capture, _sanitize_replay_block (ordered-blocks replay), and
|
||||
_convert_content_part_to_anthropic (content-list replay).
|
||||
"""
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.expanduser("~/.hermes/hermes-agent"))
|
||||
|
||||
import pytest
|
||||
from agent.anthropic_adapter import (
|
||||
_sanitize_replay_block,
|
||||
_convert_content_part_to_anthropic,
|
||||
_convert_assistant_message,
|
||||
)
|
||||
|
||||
FORBIDDEN = {"parsed_output", "caller"}
|
||||
|
||||
|
||||
def _assert_clean(block):
|
||||
"""No forbidden output-only key, and no null citations, anywhere."""
|
||||
assert isinstance(block, dict)
|
||||
for k in FORBIDDEN:
|
||||
assert k not in block, f"forbidden field {k!r} survived: {block}"
|
||||
if "citations" in block:
|
||||
assert isinstance(block["citations"], list) and block["citations"], \
|
||||
"citations must be a non-empty list if present (None/[] is input-invalid)"
|
||||
|
||||
|
||||
class TestSanitizeReplayBlock:
|
||||
def test_text_block_strips_parsed_output_and_null_citations(self):
|
||||
poisoned = {"type": "text", "text": "hi", "parsed_output": None, "citations": None}
|
||||
out = _sanitize_replay_block(poisoned)
|
||||
_assert_clean(out)
|
||||
assert out == {"type": "text", "text": "hi"}
|
||||
|
||||
def test_tool_use_strips_caller(self):
|
||||
poisoned = {"type": "tool_use", "id": "toolu_1", "name": "read_file",
|
||||
"input": {"path": "a"}, "caller": {"type": "agent"}}
|
||||
out = _sanitize_replay_block(poisoned)
|
||||
_assert_clean(out)
|
||||
assert out["name"] == "read_file" and out["input"] == {"path": "a"}
|
||||
|
||||
def test_thinking_preserves_signature(self):
|
||||
b = {"type": "thinking", "thinking": "x", "signature": "sig-AAA"}
|
||||
out = _sanitize_replay_block(b)
|
||||
assert out == {"type": "thinking", "thinking": "x", "signature": "sig-AAA"}
|
||||
|
||||
def test_text_keeps_real_citations(self):
|
||||
real = [{"type": "char_location", "cited_text": "q"}]
|
||||
out = _sanitize_replay_block({"type": "text", "text": "t", "citations": real})
|
||||
assert out["citations"] == real
|
||||
|
||||
def test_unknown_type_dropped(self):
|
||||
assert _sanitize_replay_block({"type": "server_tool_use", "foo": 1}) is None
|
||||
|
||||
|
||||
class TestContentPartConversion:
|
||||
def test_stored_text_block_with_parsed_output_cleaned(self):
|
||||
# The exact content.N.text.parsed_output failure shape.
|
||||
part = {"type": "text", "text": "hello", "parsed_output": None, "citations": None}
|
||||
out = _convert_content_part_to_anthropic(part)
|
||||
_assert_clean(out)
|
||||
|
||||
|
||||
class TestAssistantReplay:
|
||||
def test_interleaved_blocks_replayed_clean_and_ordered(self):
|
||||
m = {
|
||||
"role": "assistant",
|
||||
"anthropic_content_blocks": [
|
||||
{"type": "thinking", "thinking": "plan", "signature": "s1"},
|
||||
{"type": "text", "text": "doing it", "parsed_output": None, "citations": None},
|
||||
{"type": "tool_use", "id": "toolu_1", "name": "read_file",
|
||||
"input": {"path": "a"}, "caller": {"type": "agent"}},
|
||||
],
|
||||
}
|
||||
out = _convert_assistant_message(m)
|
||||
blocks = out["content"]
|
||||
# order preserved
|
||||
assert [b["type"] for b in blocks] == ["thinking", "text", "tool_use"]
|
||||
# every block clean
|
||||
for b in blocks:
|
||||
_assert_clean(b)
|
||||
# signature + tool fields intact
|
||||
assert blocks[0]["signature"] == "s1"
|
||||
assert blocks[2]["name"] == "read_file"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(pytest.main([__file__, "-v"]))
|
||||
314
tests/agent/test_anthropic_thinking_block_order.py
Normal file
314
tests/agent/test_anthropic_thinking_block_order.py
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
"""Regression test for the Anthropic interleaved thinking-block 400.
|
||||
|
||||
Reproduces: HTTP 400 ``messages.N.content.M: thinking or redacted_thinking
|
||||
blocks in the latest assistant message cannot be modified. These blocks must
|
||||
remain as they were in the original response.``
|
||||
|
||||
Root cause under test
|
||||
----------------------
|
||||
With adaptive / interleaved thinking (Claude 4.6+, e.g. Opus 4.8), a single
|
||||
assistant turn can emit content blocks in an interleaved order::
|
||||
|
||||
thinking_1 (signed) · tool_use_1 · thinking_2 (signed) · tool_use_2
|
||||
|
||||
Anthropic signs each thinking block against the turn content that precedes it
|
||||
at its position. ``thinking_2`` is signed with ``tool_use_1`` before it.
|
||||
|
||||
``AnthropicTransport.normalize_response`` (agent/transports/anthropic.py)
|
||||
splits the turn into two *parallel* lists — ``reasoning_details`` (thinking
|
||||
blocks) and ``tool_calls`` (tool_use blocks) — discarding the cross-type
|
||||
ordering. ``run_agent`` stores those as separate fields on the assistant
|
||||
message. On replay, ``_convert_assistant_message`` (agent/anthropic_adapter.py)
|
||||
rebuilds the content as ``[all thinking][text][all tool_use]``, which reorders
|
||||
``thinking_2`` ahead of ``tool_use_1``. The signature no longer matches its
|
||||
original position, so Anthropic rejects the latest assistant message with the
|
||||
400 above.
|
||||
|
||||
This test asserts that an interleaved turn round-trips through
|
||||
normalize_response -> stored message -> convert_messages_to_anthropic with its
|
||||
block order preserved. It FAILS on the current code (documenting the bug) and
|
||||
should PASS once block ordering is preserved on replay.
|
||||
"""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.transports import get_transport
|
||||
from agent.anthropic_adapter import convert_messages_to_anthropic
|
||||
|
||||
|
||||
def _thinking_block(text: str, signature: str) -> SimpleNamespace:
|
||||
"""A signed Anthropic thinking block, shaped like the SDK object."""
|
||||
return SimpleNamespace(type="thinking", thinking=text, signature=signature)
|
||||
|
||||
|
||||
def _tool_use_block(block_id: str, name: str, payload: dict) -> SimpleNamespace:
|
||||
return SimpleNamespace(type="tool_use", id=block_id, name=name, input=payload)
|
||||
|
||||
|
||||
def _interleaved_response() -> SimpleNamespace:
|
||||
"""An assistant turn with thinking interleaved between two tool_use blocks."""
|
||||
return SimpleNamespace(
|
||||
content=[
|
||||
_thinking_block("Plan: inspect file A first.", "sig-AAA"),
|
||||
_tool_use_block("toolu_1", "read_file", {"path": "a.py"}),
|
||||
_thinking_block("A looked fine; now inspect B.", "sig-BBB"),
|
||||
_tool_use_block("toolu_2", "read_file", {"path": "b.py"}),
|
||||
],
|
||||
stop_reason="tool_use",
|
||||
usage=None,
|
||||
)
|
||||
|
||||
|
||||
def _stored_assistant_message(normalized) -> dict:
|
||||
"""Reconstruct the OpenAI-style assistant message the way run_agent stores it.
|
||||
|
||||
run_agent.py persists assistant turns as separate fields: content,
|
||||
reasoning_details (from provider_data), and tool_calls. See
|
||||
run_agent.py L1513-1516 and hermes_state.py.
|
||||
"""
|
||||
provider_data = normalized.provider_data or {}
|
||||
tool_calls = []
|
||||
for tc in (normalized.tool_calls or []):
|
||||
tool_calls.append({
|
||||
"id": tc.id,
|
||||
"type": "function",
|
||||
"function": {"name": tc.name, "arguments": tc.arguments},
|
||||
})
|
||||
msg = {
|
||||
"role": "assistant",
|
||||
"content": normalized.content or "",
|
||||
"reasoning_details": provider_data.get("reasoning_details"),
|
||||
"tool_calls": tool_calls,
|
||||
}
|
||||
# build_assistant_message lifts the verbatim ordered-block channel onto
|
||||
# the stored message; mirror that here.
|
||||
blocks = provider_data.get("anthropic_content_blocks")
|
||||
if blocks:
|
||||
msg["anthropic_content_blocks"] = blocks
|
||||
return msg
|
||||
|
||||
|
||||
def _original_block_order(response) -> list:
|
||||
"""The (type, key) sequence of the original interleaved response."""
|
||||
order = []
|
||||
for b in response.content:
|
||||
if b.type == "thinking":
|
||||
order.append(("thinking", b.signature))
|
||||
elif b.type == "tool_use":
|
||||
order.append(("tool_use", b.id))
|
||||
return order
|
||||
|
||||
|
||||
def _replayed_block_order(assistant_content) -> list:
|
||||
order = []
|
||||
for b in assistant_content:
|
||||
if not isinstance(b, dict):
|
||||
continue
|
||||
if b.get("type") in ("thinking", "redacted_thinking"):
|
||||
order.append(("thinking", b.get("signature")))
|
||||
elif b.get("type") == "tool_use":
|
||||
order.append(("tool_use", b.get("id")))
|
||||
return order
|
||||
|
||||
|
||||
class TestInterleavedThinkingBlockOrder:
|
||||
def test_normalize_response_loses_interleaving(self):
|
||||
"""Confirm the lossy split: normalize_response stores thinking and
|
||||
tool_use in independent fields with no positional linkage."""
|
||||
transport = get_transport("anthropic_messages")
|
||||
normalized = transport.normalize_response(_interleaved_response())
|
||||
|
||||
# Both thinking blocks are captured...
|
||||
details = (normalized.provider_data or {}).get("reasoning_details")
|
||||
assert details is not None and len(details) == 2
|
||||
# ...and both tool calls...
|
||||
assert normalized.tool_calls is not None and len(normalized.tool_calls) == 2
|
||||
# ...but they live in separate fields. There is no single ordered
|
||||
# structure recording that thinking_2 sat between the two tool calls.
|
||||
# (This is the structural precondition for the reorder bug.)
|
||||
|
||||
def test_interleaved_order_preserved_on_replay(self):
|
||||
"""The latest assistant message must replay blocks in their ORIGINAL
|
||||
order, or Anthropic rejects the signed thinking blocks with a 400.
|
||||
|
||||
FAILS on current code: _convert_assistant_message front-loads all
|
||||
thinking blocks, producing
|
||||
thinking_1 · thinking_2 · tool_use_1 · tool_use_2
|
||||
instead of the original
|
||||
thinking_1 · tool_use_1 · thinking_2 · tool_use_2
|
||||
"""
|
||||
response = _interleaved_response()
|
||||
original_order = _original_block_order(response)
|
||||
|
||||
transport = get_transport("anthropic_messages")
|
||||
normalized = transport.normalize_response(response)
|
||||
assistant_msg = _stored_assistant_message(normalized)
|
||||
|
||||
# Build a minimal conversation where this assistant turn is the LATEST
|
||||
# assistant message (the one whose signed blocks are sent verbatim).
|
||||
messages = [
|
||||
{"role": "user", "content": "Inspect a.py and b.py."},
|
||||
assistant_msg,
|
||||
{"role": "tool", "tool_call_id": "toolu_1", "content": "a.py: ok"},
|
||||
{"role": "tool", "tool_call_id": "toolu_2", "content": "b.py: ok"},
|
||||
]
|
||||
|
||||
_system, anthropic_messages = convert_messages_to_anthropic(
|
||||
messages,
|
||||
base_url=None, # direct Anthropic
|
||||
model="claude-opus-4-8", # adaptive thinking family
|
||||
)
|
||||
|
||||
# Find the (latest) assistant message in the converted output.
|
||||
assistant_out = [m for m in anthropic_messages if m.get("role") == "assistant"]
|
||||
assert assistant_out, "no assistant message in converted output"
|
||||
replayed_order = _replayed_block_order(assistant_out[-1]["content"])
|
||||
|
||||
assert replayed_order == original_order, (
|
||||
"Interleaved thinking/tool_use order was not preserved on replay.\n"
|
||||
f" original: {original_order}\n"
|
||||
f" replayed: {replayed_order}\n"
|
||||
"Anthropic signs thinking blocks against their original position; "
|
||||
"reordering invalidates the signature -> HTTP 400 'thinking blocks "
|
||||
"in the latest assistant message cannot be modified'."
|
||||
)
|
||||
|
||||
def test_replay_falls_back_gracefully_without_ordered_blocks(self):
|
||||
"""Without the ordered-block channel, conversion must not crash.
|
||||
|
||||
The channel is intentionally NOT persisted to state.db (in-memory
|
||||
only): a session reloaded from disk after a crash loses the field
|
||||
and falls back to reconstruction. That replay may take one HTTP 400,
|
||||
which the thinking-signature recovery (#43667) absorbs by stripping
|
||||
reasoning_details and retrying. This test pins the fallback shape:
|
||||
conversion still produces a valid assistant message from the
|
||||
parallel reasoning_details + tool_calls fields.
|
||||
"""
|
||||
response = _interleaved_response()
|
||||
transport = get_transport("anthropic_messages")
|
||||
normalized = transport.normalize_response(response)
|
||||
assistant_msg = _stored_assistant_message(normalized)
|
||||
# Simulate a disk reload: the in-memory-only channel is gone.
|
||||
assistant_msg.pop("anthropic_content_blocks", None)
|
||||
|
||||
messages = [
|
||||
assistant_msg,
|
||||
{"role": "tool", "tool_call_id": "toolu_1", "content": "a ok"},
|
||||
{"role": "tool", "tool_call_id": "toolu_2", "content": "b ok"},
|
||||
]
|
||||
_system, anthropic_messages = convert_messages_to_anthropic(
|
||||
messages, base_url=None, model="claude-opus-4-8",
|
||||
)
|
||||
assistant_out = [m for m in anthropic_messages if m.get("role") == "assistant"]
|
||||
assert assistant_out, "no assistant message in converted output"
|
||||
content = assistant_out[-1]["content"]
|
||||
assert isinstance(content, list) and content, "fallback produced empty content"
|
||||
# Reconstruction keeps both tool_use blocks (answered by results).
|
||||
tool_ids = [b.get("id") for b in content if isinstance(b, dict) and b.get("type") == "tool_use"]
|
||||
assert set(tool_ids) == {"toolu_1", "toolu_2"}
|
||||
|
||||
|
||||
class TestInterleavedReplayCredentialRedaction:
|
||||
"""The verbatim-replay fast path must not leak un-redacted secrets.
|
||||
|
||||
anthropic_content_blocks captures each tool_use ``input`` from the RAW API
|
||||
response (normalize_response), which is NOT credential-redacted. The
|
||||
parallel tool_calls[].function.arguments IS redacted at storage time
|
||||
(build_assistant_message, #19798). If the fast path replays the block's raw
|
||||
input verbatim, a secret the model inlined into a tool call rides back onto
|
||||
the wire — even though it is redacted everywhere else in history. The fix
|
||||
re-sources tool_use input from the redacted tool_calls map by id.
|
||||
"""
|
||||
|
||||
def test_tool_use_input_resourced_from_redacted_tool_calls(self):
|
||||
REDACTED = "[REDACTED_SECRET]"
|
||||
# Ordered channel: raw input carries the live secret (as captured from
|
||||
# the unredacted API response).
|
||||
ordered = [
|
||||
{"type": "thinking", "thinking": "Call the API.", "signature": "sig-AAA"},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_1",
|
||||
"name": "terminal",
|
||||
"input": {"command": "curl -H 'Authorization: Bearer sk-LIVE-SECRET-123'"},
|
||||
},
|
||||
{"type": "thinking", "thinking": "Now the second call.", "signature": "sig-BBB"},
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_2",
|
||||
"name": "terminal",
|
||||
"input": {"command": "echo done"},
|
||||
},
|
||||
]
|
||||
# Stored tool_calls: arguments already redacted (the #19798 path).
|
||||
assistant_msg = {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_details": [b for b in ordered if b["type"] == "thinking"],
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "toolu_1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "terminal",
|
||||
"arguments": json.dumps(
|
||||
{"command": f"curl -H 'Authorization: Bearer {REDACTED}'"}
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "toolu_2",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "terminal",
|
||||
"arguments": json.dumps({"command": "echo done"}),
|
||||
},
|
||||
},
|
||||
],
|
||||
"anthropic_content_blocks": ordered,
|
||||
}
|
||||
messages = [
|
||||
{"role": "user", "content": "Hit the API twice."},
|
||||
assistant_msg,
|
||||
{"role": "tool", "tool_call_id": "toolu_1", "content": "200 OK"},
|
||||
{"role": "tool", "tool_call_id": "toolu_2", "content": "done"},
|
||||
]
|
||||
|
||||
_system, anthropic_messages = convert_messages_to_anthropic(
|
||||
messages, base_url=None, model="claude-opus-4-8",
|
||||
)
|
||||
assistant_out = [m for m in anthropic_messages if m.get("role") == "assistant"]
|
||||
assert assistant_out, "no assistant message in converted output"
|
||||
blocks = assistant_out[-1]["content"]
|
||||
|
||||
tool_uses = {b["id"]: b for b in blocks if b.get("type") == "tool_use"}
|
||||
assert set(tool_uses) == {"toolu_1", "toolu_2"}, "tool_use blocks missing/renamed"
|
||||
|
||||
# The replayed input must be the REDACTED value, not the live secret.
|
||||
replayed_cmd = tool_uses["toolu_1"]["input"]["command"]
|
||||
assert "sk-LIVE-SECRET-123" not in replayed_cmd, (
|
||||
"Un-redacted secret leaked onto the wire via the verbatim-replay "
|
||||
"fast path. tool_use input must be re-sourced from the redacted "
|
||||
"tool_calls map, not the raw captured block."
|
||||
)
|
||||
assert REDACTED in replayed_cmd
|
||||
|
||||
# Interleave order is still preserved (the reason the channel exists).
|
||||
order = [
|
||||
("thinking", b.get("signature")) if b.get("type") == "thinking"
|
||||
else ("tool_use", b.get("id"))
|
||||
for b in blocks if b.get("type") in ("thinking", "tool_use")
|
||||
]
|
||||
assert order == [
|
||||
("thinking", "sig-AAA"),
|
||||
("tool_use", "toolu_1"),
|
||||
("thinking", "sig-BBB"),
|
||||
("tool_use", "toolu_2"),
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(pytest.main([__file__, "-v"]))
|
||||
|
|
@ -1471,3 +1471,127 @@ class TestCallConverseInvalidatesOnStaleError:
|
|||
)
|
||||
|
||||
assert _bedrock_runtime_client_cache.get("us-east-1") is live_client
|
||||
|
||||
|
||||
class TestStreamingAccessDeniedDetection:
|
||||
"""is_streaming_access_denied_error() recognizes IAM denials of
|
||||
bedrock:InvokeModelWithResponseStream (InvokeModel-only policies)."""
|
||||
|
||||
def _denied_client_error(self):
|
||||
from botocore.exceptions import ClientError
|
||||
return ClientError(
|
||||
error_response={
|
||||
"Error": {
|
||||
"Code": "AccessDeniedException",
|
||||
"Message": (
|
||||
"User: arn:aws:iam::123456789012:user/x is not "
|
||||
"authorized to perform: "
|
||||
"bedrock:InvokeModelWithResponseStream on resource: "
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/"
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0"
|
||||
),
|
||||
}
|
||||
},
|
||||
operation_name="ConverseStream",
|
||||
)
|
||||
|
||||
def test_matches_access_denied_client_error(self):
|
||||
pytest.importorskip("botocore", reason="botocore required for Bedrock exception tests")
|
||||
from agent.bedrock_adapter import is_streaming_access_denied_error
|
||||
assert is_streaming_access_denied_error(self._denied_client_error()) is True
|
||||
|
||||
def test_ignores_access_denied_for_other_actions(self):
|
||||
"""AccessDenied on InvokeModel itself is NOT a streaming-only denial."""
|
||||
pytest.importorskip("botocore", reason="botocore required for Bedrock exception tests")
|
||||
from agent.bedrock_adapter import is_streaming_access_denied_error
|
||||
from botocore.exceptions import ClientError
|
||||
exc = ClientError(
|
||||
error_response={
|
||||
"Error": {
|
||||
"Code": "AccessDeniedException",
|
||||
"Message": (
|
||||
"User is not authorized to perform: bedrock:InvokeModel"
|
||||
),
|
||||
}
|
||||
},
|
||||
operation_name="Converse",
|
||||
)
|
||||
assert is_streaming_access_denied_error(exc) is False
|
||||
|
||||
def test_ignores_validation_error_mentioning_action(self):
|
||||
"""Non-authz ClientErrors don't match even if the action name appears."""
|
||||
pytest.importorskip("botocore", reason="botocore required for Bedrock exception tests")
|
||||
from agent.bedrock_adapter import is_streaming_access_denied_error
|
||||
from botocore.exceptions import ClientError
|
||||
exc = ClientError(
|
||||
error_response={
|
||||
"Error": {
|
||||
"Code": "ValidationException",
|
||||
"Message": "InvokeModelWithResponseStream input malformed",
|
||||
}
|
||||
},
|
||||
operation_name="ConverseStream",
|
||||
)
|
||||
assert is_streaming_access_denied_error(exc) is False
|
||||
|
||||
def test_matches_wrapped_sdk_permission_error(self):
|
||||
"""Non-ClientError wrappers (AnthropicBedrock SDK) match on message."""
|
||||
from agent.bedrock_adapter import is_streaming_access_denied_error
|
||||
exc = RuntimeError(
|
||||
"PermissionDeniedError: user is not authorized to perform: "
|
||||
"bedrock:InvokeModelWithResponseStream"
|
||||
)
|
||||
assert is_streaming_access_denied_error(exc) is True
|
||||
|
||||
def test_ignores_unrelated_errors(self):
|
||||
from agent.bedrock_adapter import is_streaming_access_denied_error
|
||||
assert is_streaming_access_denied_error(ValueError("boom")) is False
|
||||
assert is_streaming_access_denied_error(
|
||||
RuntimeError("stream not supported")
|
||||
) is False
|
||||
|
||||
|
||||
class TestCallConverseStreamIamFallback:
|
||||
"""call_converse_stream() falls back to converse() when IAM denies the
|
||||
streaming action — InvokeModel-only policies keep working."""
|
||||
|
||||
def test_falls_back_to_converse_on_streaming_denial(self):
|
||||
pytest.importorskip("botocore", reason="botocore required for Bedrock exception tests")
|
||||
from agent.bedrock_adapter import (
|
||||
_bedrock_runtime_client_cache,
|
||||
call_converse_stream,
|
||||
reset_client_cache,
|
||||
)
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
reset_client_cache()
|
||||
client = MagicMock()
|
||||
client.converse_stream.side_effect = ClientError(
|
||||
error_response={
|
||||
"Error": {
|
||||
"Code": "AccessDeniedException",
|
||||
"Message": (
|
||||
"User is not authorized to perform: "
|
||||
"bedrock:InvokeModelWithResponseStream"
|
||||
),
|
||||
}
|
||||
},
|
||||
operation_name="ConverseStream",
|
||||
)
|
||||
client.converse.return_value = {
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "hi"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2},
|
||||
}
|
||||
_bedrock_runtime_client_cache["us-east-1"] = client
|
||||
|
||||
result = call_converse_stream(
|
||||
region="us-east-1",
|
||||
model="anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
client.converse.assert_called_once()
|
||||
assert result.choices[0].message.content == "hi"
|
||||
# Not a stale connection — client stays cached.
|
||||
assert _bedrock_runtime_client_cache.get("us-east-1") is client
|
||||
|
|
|
|||
405
tests/agent/test_coding_context.py
Normal file
405
tests/agent/test_coding_context.py
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
"""Tests for agent.coding_context — RuntimeMode seam, resolver, toolset, git probe."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent import coding_context as cc
|
||||
|
||||
|
||||
def _git_init(path):
|
||||
env = {
|
||||
"GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t",
|
||||
"GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t",
|
||||
}
|
||||
for args in (
|
||||
["init", "-q", "-b", "main"],
|
||||
["commit", "-q", "--allow-empty", "-m", "init commit"],
|
||||
):
|
||||
subprocess.run(["git", "-C", str(path), *args], check=True, env={**env, "HOME": str(path)})
|
||||
|
||||
|
||||
# ── resolver ──────────────────────────────────────────────────────────────
|
||||
|
||||
class TestIsCodingContext:
|
||||
def test_off_never_activates(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
cfg = {"agent": {"coding_context": "off"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=tmp_path, config=cfg) is False
|
||||
|
||||
def test_on_forces_even_without_git(self, tmp_path):
|
||||
cfg = {"agent": {"coding_context": "on"}}
|
||||
assert cc.is_coding_context(platform="telegram", cwd=tmp_path, config=cfg) is True
|
||||
|
||||
def test_auto_requires_git_repo(self, tmp_path):
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=tmp_path, config=cfg) is False
|
||||
_git_init(tmp_path)
|
||||
assert cc.is_coding_context(platform="cli", cwd=tmp_path, config=cfg) is True
|
||||
|
||||
def test_auto_skips_messaging_surfaces(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.is_coding_context(platform="discord", cwd=tmp_path, config=cfg) is False
|
||||
assert cc.is_coding_context(platform="tui", cwd=tmp_path, config=cfg) is True
|
||||
|
||||
def test_default_mode_is_auto(self, tmp_path):
|
||||
# Unknown/missing value normalizes to auto.
|
||||
_git_init(tmp_path)
|
||||
assert cc.is_coding_context(platform="cli", cwd=tmp_path, config={}) is True
|
||||
|
||||
|
||||
# ── toolset substitution ────────────────────────────────────────────────────
|
||||
|
||||
class TestCodingSelection:
|
||||
def test_selects_coding_under_focus(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
cfg = {"agent": {"coding_context": "focus"}}
|
||||
out = cc.coding_selection(platform="cli", cwd=tmp_path, config=cfg)
|
||||
assert out is not None
|
||||
assert out[0] == cc.CODING_TOOLSET
|
||||
|
||||
def test_auto_is_prompt_only(self, tmp_path):
|
||||
# Default posture must never override the user's configured toolsets —
|
||||
# off-by-default toolsets are already off, and explicit opt-ins
|
||||
# (image-gen, spotify, …) survive entering a code workspace.
|
||||
_git_init(tmp_path)
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.coding_selection(platform="cli", cwd=tmp_path, config=cfg) is None
|
||||
# …while the prompt posture is still active.
|
||||
assert cc.is_coding_context(platform="cli", cwd=tmp_path, config=cfg) is True
|
||||
|
||||
def test_on_is_prompt_only(self, tmp_path):
|
||||
cfg = {"agent": {"coding_context": "on"}}
|
||||
assert cc.coding_selection(platform="cli", cwd=tmp_path, config=cfg) is None
|
||||
assert cc.is_coding_context(platform="cli", cwd=tmp_path, config=cfg) is True
|
||||
|
||||
def test_focus_requires_workspace(self, tmp_path):
|
||||
# focus inherits auto's detection gate — bare dir stays general.
|
||||
cfg = {"agent": {"coding_context": "focus"}}
|
||||
assert cc.coding_selection(platform="cli", cwd=tmp_path, config=cfg) is None
|
||||
|
||||
def test_none_when_inactive(self, tmp_path):
|
||||
cfg = {"agent": {"coding_context": "off"}}
|
||||
assert cc.coding_selection(platform="cli", cwd=tmp_path, config=cfg) is None
|
||||
|
||||
def test_coding_toolset_is_registered(self):
|
||||
from toolsets import resolve_toolset
|
||||
|
||||
tools = resolve_toolset(cc.CODING_TOOLSET)
|
||||
# Coding essentials present…
|
||||
for t in ("read_file", "write_file", "patch", "search_files", "terminal", "todo"):
|
||||
assert t in tools
|
||||
# …and the noise is gone.
|
||||
for t in ("send_message", "text_to_speech", "image_generate", "computer_use"):
|
||||
assert t not in tools
|
||||
|
||||
|
||||
# ── git/workspace probe ─────────────────────────────────────────────────────
|
||||
|
||||
class TestWorkspaceBlock:
|
||||
def test_empty_outside_repo(self, tmp_path):
|
||||
assert cc.build_coding_workspace_block(tmp_path) == ""
|
||||
|
||||
def test_reports_branch_and_clean_status(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
block = cc.build_coding_workspace_block(tmp_path)
|
||||
assert "Workspace" in block
|
||||
assert f"Root: {tmp_path.resolve()}" in block or "Root:" in block
|
||||
assert "Branch: main" in block
|
||||
assert "Status: clean" in block
|
||||
assert "init commit" in block
|
||||
|
||||
def test_reports_dirty_counts(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
(tmp_path / "untracked.txt").write_text("hi")
|
||||
block = cc.build_coding_workspace_block(tmp_path)
|
||||
assert "untracked" in block
|
||||
assert "clean" not in block.split("Status:")[1].splitlines()[0]
|
||||
|
||||
|
||||
# ── project facts (verify-loop detection) ───────────────────────────────────
|
||||
|
||||
class TestProjectFacts:
|
||||
def test_package_json_scripts_surface_verify_commands(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
(tmp_path / "package.json").write_text(
|
||||
json.dumps({"scripts": {"test": "vitest", "lint": "eslint .", "dev": "vite"}})
|
||||
)
|
||||
(tmp_path / "pnpm-lock.yaml").write_text("")
|
||||
block = cc.build_coding_workspace_block(tmp_path)
|
||||
assert "Project: package.json (pnpm)" in block
|
||||
assert "pnpm run test" in block and "pnpm run lint" in block
|
||||
# Non-verify scripts (dev servers, …) stay out of the snapshot.
|
||||
assert "run dev" not in block
|
||||
|
||||
def test_pytest_config_and_run_tests_script(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
(tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\n")
|
||||
scripts = tmp_path / "scripts"
|
||||
scripts.mkdir()
|
||||
(scripts / "run_tests.sh").write_text("#!/bin/sh\n")
|
||||
block = cc.build_coding_workspace_block(tmp_path)
|
||||
assert "scripts/run_tests.sh" in block
|
||||
assert "pytest" in block.split("Verify:")[1]
|
||||
|
||||
def test_makefile_verify_targets_only(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
(tmp_path / "Makefile").write_text("test:\n\tgo test ./...\n\ndeploy:\n\t./deploy.sh\n")
|
||||
block = cc.build_coding_workspace_block(tmp_path)
|
||||
assert "make test" in block
|
||||
assert "make deploy" not in block
|
||||
|
||||
def test_context_files_listed(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
(tmp_path / "AGENTS.md").write_text("# rules")
|
||||
block = cc.build_coding_workspace_block(tmp_path)
|
||||
assert "Context files: AGENTS.md" in block
|
||||
|
||||
def test_marker_only_project_gets_snapshot_without_git(self, tmp_path):
|
||||
# A non-git project (manifest only) still gets a workspace snapshot —
|
||||
# just without the git lines.
|
||||
(tmp_path / "package.json").write_text("{}")
|
||||
block = cc.build_coding_workspace_block(tmp_path)
|
||||
assert f"Root: {tmp_path.resolve()}" in block
|
||||
assert "package.json" in block
|
||||
assert "Branch:" not in block and "Status:" not in block
|
||||
|
||||
def test_malformed_package_json_is_ignored(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
(tmp_path / "package.json").write_text("{not json")
|
||||
block = cc.build_coding_workspace_block(tmp_path)
|
||||
assert "Project: package.json" in block
|
||||
assert "Verify:" not in block
|
||||
|
||||
|
||||
# ── $HOME dotfiles guard ────────────────────────────────────────────────────
|
||||
|
||||
class TestHomeDotfilesGuard:
|
||||
def test_dotfiles_repo_at_home_is_not_coding(self, tmp_path, monkeypatch):
|
||||
home = tmp_path / "home"
|
||||
home.mkdir()
|
||||
_git_init(home)
|
||||
monkeypatch.setattr(Path, "home", lambda: home)
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=home, config=cfg) is False
|
||||
# …and a plain subdirectory of the dotfiles repo stays general too.
|
||||
docs = home / "Documents"
|
||||
docs.mkdir()
|
||||
assert cc.is_coding_context(platform="cli", cwd=docs, config=cfg) is False
|
||||
|
||||
def test_marker_at_home_is_not_a_project_signal(self, tmp_path, monkeypatch):
|
||||
home = tmp_path / "home"
|
||||
home.mkdir()
|
||||
(home / "Makefile").write_text("all:\n")
|
||||
monkeypatch.setattr(Path, "home", lambda: home)
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=home, config=cfg) is False
|
||||
|
||||
def test_real_project_under_dotfiles_home_still_detects(self, tmp_path, monkeypatch):
|
||||
home = tmp_path / "home"
|
||||
home.mkdir()
|
||||
_git_init(home)
|
||||
monkeypatch.setattr(Path, "home", lambda: home)
|
||||
proj = home / "www" / "app"
|
||||
proj.mkdir(parents=True)
|
||||
(proj / "package.json").write_text("{}")
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=proj, config=cfg) is True
|
||||
|
||||
def test_on_mode_bypasses_the_guard(self, tmp_path, monkeypatch):
|
||||
home = tmp_path / "home"
|
||||
home.mkdir()
|
||||
monkeypatch.setattr(Path, "home", lambda: home)
|
||||
cfg = {"agent": {"coding_context": "on"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=home, config=cfg) is True
|
||||
|
||||
|
||||
# ── prompt assembly integration ─────────────────────────────────────────────
|
||||
|
||||
class TestStatusParsing:
|
||||
def test_parse_status_counts_and_branch(self):
|
||||
porcelain = (
|
||||
"# branch.head feature\n"
|
||||
"# branch.upstream origin/feature\n"
|
||||
"# branch.ab +2 -1\n"
|
||||
"1 M. N... 100644 100644 100644 aaa bbb staged.py\n"
|
||||
"1 .M N... 100644 100644 100644 ccc ddd modified.py\n"
|
||||
"? new.py\n"
|
||||
"u UU N... 1 2 3 abc def conflict.py\n"
|
||||
)
|
||||
branch, counts = cc._parse_status(porcelain)
|
||||
assert branch["head"] == "feature"
|
||||
assert branch["upstream"] == "origin/feature"
|
||||
assert branch["ahead"] == "2" and branch["behind"] == "1"
|
||||
assert counts["staged"] == 1
|
||||
assert counts["modified"] == 1
|
||||
assert counts["untracked"] == 1
|
||||
assert counts["conflicts"] == 1
|
||||
|
||||
|
||||
# ── RuntimeMode seam ────────────────────────────────────────────────────────
|
||||
|
||||
class TestRuntimeMode:
|
||||
def test_resolves_coding_in_repo(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
mode = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={})
|
||||
assert mode.is_coding is True
|
||||
assert mode.kind == "coding"
|
||||
assert mode.profile is cc.CODING_PROFILE
|
||||
|
||||
def test_resolves_general_outside_workspace(self, tmp_path):
|
||||
mode = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={})
|
||||
assert mode.is_coding is False
|
||||
assert mode.kind == "general"
|
||||
# General posture pins no toolset and injects no blocks.
|
||||
assert mode.toolset_selection() is None
|
||||
assert mode.system_blocks() == []
|
||||
|
||||
def test_is_frozen(self, tmp_path):
|
||||
mode = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={})
|
||||
with pytest.raises(Exception):
|
||||
mode.profile = cc.CODING_PROFILE # type: ignore[misc]
|
||||
|
||||
def test_system_blocks_include_brief_and_workspace(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
mode = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={"agent": {"coding_context": "on"}})
|
||||
blocks = mode.system_blocks()
|
||||
assert any("coding agent" in b for b in blocks)
|
||||
assert any("Workspace" in b for b in blocks)
|
||||
|
||||
def test_toolset_selection_gated_on_focus(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
focus = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={"agent": {"coding_context": "focus"}})
|
||||
sel = focus.toolset_selection()
|
||||
assert sel and sel[0] == cc.CODING_TOOLSET
|
||||
# auto/on resolve the coding profile but stay prompt-only.
|
||||
for raw in ("auto", "on"):
|
||||
mode = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={"agent": {"coding_context": raw}})
|
||||
assert mode.is_coding is True
|
||||
assert mode.toolset_selection() is None
|
||||
|
||||
|
||||
# ── edit-format steering (per-model harness tuning) ──────────────────────────
|
||||
|
||||
class TestEditFormatSteering:
|
||||
def test_family_detection(self):
|
||||
assert cc._model_family("openai/gpt-5.4") == "patch"
|
||||
assert cc._model_family("openai/codex-mini") == "patch"
|
||||
assert cc._model_family("anthropic/claude-opus-4.8") == "replace"
|
||||
assert cc._model_family("anthropic/claude-sonnet-4") == "replace"
|
||||
# Gemini + open-weight coding models (RL'd on str_replace-style
|
||||
# editors) steer to replace, not neutral.
|
||||
for m in (
|
||||
"google/gemini-3-pro", "deepseek-v3.2", "qwen3-coder",
|
||||
"moonshot/kimi-k2", "zai/glm-4.6", "nousresearch/hermes-4-405b",
|
||||
):
|
||||
assert cc._model_family(m) == "replace"
|
||||
# Unknown family and no model both fall through to neutral wording.
|
||||
assert cc._model_family("acme/foo-1") is None
|
||||
assert cc._model_family(None) is None
|
||||
assert cc._model_family("") is None
|
||||
|
||||
def test_openai_family_gets_v4a_nudge(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
mode = cc.resolve_runtime_mode(
|
||||
platform="cli", cwd=tmp_path,
|
||||
config={"agent": {"coding_context": "on"}}, model="openai/gpt-5.4",
|
||||
)
|
||||
brief = mode.system_blocks()[0]
|
||||
assert "mode='patch'" in brief
|
||||
assert "V4A" in brief
|
||||
assert "write_file" in brief # new files authored, not patched
|
||||
|
||||
def test_anthropic_family_gets_replace_nudge(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
mode = cc.resolve_runtime_mode(
|
||||
platform="cli", cwd=tmp_path,
|
||||
config={"agent": {"coding_context": "on"}},
|
||||
model="anthropic/claude-opus-4.8",
|
||||
)
|
||||
brief = mode.system_blocks()[0]
|
||||
assert "mode='replace'" in brief
|
||||
assert "write_file" in brief # new files authored, not patched
|
||||
|
||||
def test_unknown_model_keeps_neutral_brief(self, tmp_path):
|
||||
# No edit-format line appended — brief equals the bare profile guidance.
|
||||
_git_init(tmp_path)
|
||||
mode = cc.resolve_runtime_mode(
|
||||
platform="cli", cwd=tmp_path,
|
||||
config={"agent": {"coding_context": "on"}}, model="acme/foo-1",
|
||||
)
|
||||
assert mode.system_blocks()[0] == cc.CODING_AGENT_GUIDANCE
|
||||
|
||||
def test_no_model_keeps_neutral_brief(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
mode = cc.resolve_runtime_mode(
|
||||
platform="cli", cwd=tmp_path,
|
||||
config={"agent": {"coding_context": "on"}},
|
||||
)
|
||||
assert mode.system_blocks()[0] == cc.CODING_AGENT_GUIDANCE
|
||||
|
||||
def test_general_posture_emits_nothing_regardless_of_model(self, tmp_path):
|
||||
# Edit steering only fires inside the coding posture.
|
||||
mode = cc.resolve_runtime_mode(
|
||||
platform="telegram", cwd=tmp_path, config={}, model="openai/gpt-5.4",
|
||||
)
|
||||
assert mode.system_blocks() == []
|
||||
|
||||
|
||||
# ── profile registry ────────────────────────────────────────────────────────
|
||||
|
||||
class TestProfiles:
|
||||
def test_registered_profiles(self):
|
||||
assert cc.get_profile("coding") is cc.CODING_PROFILE
|
||||
assert cc.get_profile("general") is cc.GENERAL_PROFILE
|
||||
|
||||
def test_unknown_profile_falls_back_to_general(self):
|
||||
assert cc.get_profile("nonsense") is cc.GENERAL_PROFILE
|
||||
|
||||
def test_coding_profile_shape(self):
|
||||
# The coding profile declares the seams other domains read.
|
||||
assert cc.CODING_PROFILE.toolset == cc.CODING_TOOLSET
|
||||
assert cc.CODING_PROFILE.guidance
|
||||
assert cc.CODING_PROFILE.model_hint == "coding"
|
||||
# General is inert.
|
||||
assert cc.GENERAL_PROFILE.toolset is None
|
||||
assert cc.GENERAL_PROFILE.guidance == ""
|
||||
|
||||
def test_skill_pruning_scoped_to_coding_posture(self, tmp_path):
|
||||
# Coding posture hides clearly-non-coding categories; coding-adjacent
|
||||
# ones stay visible (deny-list semantics).
|
||||
_git_init(tmp_path)
|
||||
coding = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={})
|
||||
hidden = coding.hidden_skill_categories()
|
||||
assert "social-media" in hidden and "smart-home" in hidden
|
||||
for kept in ("github", "devops", "software-development", "data-science"):
|
||||
assert kept not in hidden
|
||||
# General posture hides nothing.
|
||||
general = cc.resolve_runtime_mode(
|
||||
platform="telegram", cwd=tmp_path, config={}
|
||||
)
|
||||
assert general.hidden_skill_categories() == frozenset()
|
||||
|
||||
|
||||
# ── detection signals ───────────────────────────────────────────────────────
|
||||
|
||||
class TestDetection:
|
||||
@pytest.mark.parametrize("marker", ["pyproject.toml", "package.json", "go.mod", "AGENTS.md"])
|
||||
def test_project_manifest_triggers_without_git(self, tmp_path, marker):
|
||||
(tmp_path / marker).write_text("x")
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=tmp_path, config=cfg) is True
|
||||
|
||||
def test_marker_in_parent_counts_from_subdir(self, tmp_path):
|
||||
(tmp_path / "pyproject.toml").write_text("x")
|
||||
sub = tmp_path / "src" / "pkg"
|
||||
sub.mkdir(parents=True)
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=sub, config=cfg) is True
|
||||
|
||||
def test_bare_dir_is_not_coding(self, tmp_path):
|
||||
cfg = {"agent": {"coding_context": "auto"}}
|
||||
assert cc.is_coding_context(platform="cli", cwd=tmp_path, config=cfg) is False
|
||||
|
|
@ -12,6 +12,7 @@ from agent.display import (
|
|||
set_tool_preview_max_len,
|
||||
_render_inline_unified_diff,
|
||||
_summarize_rendered_diff_sections,
|
||||
_used_free_parallel,
|
||||
render_edit_diff_with_delta,
|
||||
)
|
||||
|
||||
|
|
@ -171,6 +172,46 @@ class TestCuteToolMessagePreviewLength:
|
|||
assert "[error]" not in line
|
||||
|
||||
|
||||
class TestWebProviderLabel:
|
||||
"""The free-path "Parallel search"/"Parallel fetch" verb labeling."""
|
||||
|
||||
def test_free_search_verb_is_parallel(self):
|
||||
result = json.dumps({"success": True, "data": {"web": []}, "provider": "parallel"})
|
||||
line = get_cute_tool_message("web_search", {"query": "hello"}, 0.1, result=result)
|
||||
assert "Parallel search" in line
|
||||
assert "hello" in line
|
||||
|
||||
def test_paid_search_verb_is_plain(self):
|
||||
result = json.dumps({"success": True, "data": {"web": [{"url": "u"}]}})
|
||||
line = get_cute_tool_message("web_search", {"query": "hi"}, 0.1, result=result)
|
||||
assert "Parallel" not in line
|
||||
assert "search" in line
|
||||
|
||||
def test_missing_result_verb_is_plain(self):
|
||||
line = get_cute_tool_message("web_search", {"query": "hello"}, 0.1)
|
||||
assert "Parallel" not in line
|
||||
assert "search" in line
|
||||
|
||||
def test_helper_is_parallel_free_specific(self):
|
||||
# Only Parallel's free MCP path marks results; nothing else does.
|
||||
assert _used_free_parallel(json.dumps({"provider": "parallel"})) is True
|
||||
assert _used_free_parallel(json.dumps({"provider": "exa"})) is False
|
||||
assert _used_free_parallel(json.dumps({"provider": "firecrawl"})) is False
|
||||
assert _used_free_parallel(json.dumps({"success": True, "data": {}})) is False
|
||||
assert _used_free_parallel('not json') is False
|
||||
assert _used_free_parallel(None) is False
|
||||
|
||||
def test_free_extract_verb_is_parallel(self):
|
||||
result = json.dumps({"results": [{"url": "u", "content": "x"}], "provider": "parallel"})
|
||||
line = get_cute_tool_message("web_extract", {"urls": ["https://a.test"]}, 0.1, result=result)
|
||||
assert "Parallel fetch" in line
|
||||
|
||||
def test_paid_extract_verb_is_plain(self):
|
||||
result = json.dumps({"results": [{"url": "u", "content": "x"}]})
|
||||
line = get_cute_tool_message("web_extract", {"urls": ["https://a.test"]}, 0.1, result=result)
|
||||
assert "Parallel" not in line
|
||||
|
||||
|
||||
class TestEditDiffPreview:
|
||||
def test_extract_edit_diff_for_patch(self):
|
||||
diff = extract_edit_diff("patch", '{"success": true, "diff": "--- a/x\\n+++ b/x\\n"}')
|
||||
|
|
|
|||
|
|
@ -661,6 +661,42 @@ class TestClassifyApiError:
|
|||
# Without "thinking" in the message, it shouldn't be thinking_signature
|
||||
assert result.reason != FailoverReason.thinking_signature
|
||||
|
||||
def test_anthropic_thinking_blocks_cannot_be_modified(self):
|
||||
"""Frozen-block mutation 400 (no 'signature' token) must route to
|
||||
thinking_signature recovery, not hard-abort. Regression for the
|
||||
real-world error: latest-assistant thinking blocks 'cannot be
|
||||
modified' after upstream message mutation."""
|
||||
e = MockAPIError(
|
||||
"messages.73.content.10: `thinking` or `redacted_thinking` blocks "
|
||||
"in the latest assistant message cannot be modified. These blocks "
|
||||
"must remain as they were in the original response.",
|
||||
status_code=400,
|
||||
)
|
||||
result = classify_api_error(e, provider="anthropic")
|
||||
assert result.reason == FailoverReason.thinking_signature
|
||||
assert result.retryable is True
|
||||
|
||||
def test_anthropic_thinking_cannot_be_modified_via_openrouter(self):
|
||||
"""Same frozen-block error proxied through OpenRouter must also be
|
||||
caught (provider is not gated)."""
|
||||
e = MockAPIError(
|
||||
"`thinking` or `redacted_thinking` blocks in the latest assistant "
|
||||
"message cannot be modified.",
|
||||
status_code=400,
|
||||
)
|
||||
result = classify_api_error(e, provider="openrouter")
|
||||
assert result.reason == FailoverReason.thinking_signature
|
||||
assert result.retryable is True
|
||||
|
||||
def test_400_cannot_be_modified_without_thinking_not_classified(self):
|
||||
"""A 400 'cannot be modified' that has nothing to do with thinking
|
||||
blocks must NOT be swept into thinking_signature recovery."""
|
||||
e = MockAPIError(
|
||||
"this field cannot be modified after creation", status_code=400,
|
||||
)
|
||||
result = classify_api_error(e, provider="anthropic", approx_tokens=0)
|
||||
assert result.reason != FailoverReason.thinking_signature
|
||||
|
||||
def test_invalid_encrypted_content_classified_as_retryable_replay_failure(self):
|
||||
body = {
|
||||
"error": {
|
||||
|
|
|
|||
|
|
@ -276,6 +276,42 @@ class TestBuildSkillsSystemPrompt:
|
|||
# "search" should appear only once per category
|
||||
assert result.count("- search") == 1
|
||||
|
||||
def test_hidden_categories_pruned_with_note(self, monkeypatch, tmp_path):
|
||||
"""Posture-driven pruning drops whole categories and discloses it."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
for cat, name in (("social-media", "tweet-stuff"), ("github", "pr-review")):
|
||||
d = tmp_path / "skills" / cat / name
|
||||
d.mkdir(parents=True)
|
||||
(d / "SKILL.md").write_text(
|
||||
f"---\nname: {name}\ndescription: Does {name} things\n---\n"
|
||||
)
|
||||
|
||||
result = build_skills_system_prompt(
|
||||
hidden_categories=frozenset({"social-media"})
|
||||
)
|
||||
assert "pr-review" in result
|
||||
assert "tweet-stuff" not in result
|
||||
# Disclosure note so the model knows the full catalog exists.
|
||||
assert "skills_list" in result
|
||||
|
||||
def test_hidden_categories_prune_nested_and_miss_cache_separately(
|
||||
self, monkeypatch, tmp_path
|
||||
):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
d = tmp_path / "skills" / "social-media" / "twitter" / "thread-writer"
|
||||
d.mkdir(parents=True)
|
||||
(d / "SKILL.md").write_text(
|
||||
"---\nname: thread-writer\ndescription: Write threads\n---\n"
|
||||
)
|
||||
# Nested category ("social-media/twitter") pruned via its parent.
|
||||
pruned = build_skills_system_prompt(
|
||||
hidden_categories=frozenset({"social-media"})
|
||||
)
|
||||
assert "thread-writer" not in pruned
|
||||
# Unfiltered call must not be served from the filtered cache entry.
|
||||
full = build_skills_system_prompt()
|
||||
assert "thread-writer" in full
|
||||
|
||||
def test_excludes_incompatible_platform_skills(self, monkeypatch, tmp_path):
|
||||
"""Skills with platforms: [macos] should not appear on Linux."""
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
||||
|
|
|
|||
111
tests/agent/test_stream_read_timeout_floor.py
Normal file
111
tests/agent/test_stream_read_timeout_floor.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
"""Stream read timeout must never preempt the stale-stream detector.
|
||||
|
||||
Reasoning models (e.g. Opus) routinely pause mid-stream for minutes during
|
||||
extended thinking. The stale-stream detector is deliberately scaled up to
|
||||
tolerate this (180s base, raised to 240s/300s for large contexts). The httpx
|
||||
socket read timeout, however, defaulted to a flat 120s for cloud providers and
|
||||
fired *first* — tearing down a healthy reasoning stream before the stale
|
||||
detector (which owns retry + diagnostics) could act.
|
||||
|
||||
These tests pin the invariant: for a cloud provider on the default read
|
||||
timeout, the httpx socket read timeout is floored at the stale-stream timeout
|
||||
so it can never fire before the detector. They mirror the inline logic in
|
||||
``agent/chat_completion_helpers.py`` (the real builder lives deep inside a
|
||||
worker thread, so — like ``test_local_stream_timeout.py`` — the resolution is
|
||||
reproduced here rather than driven end-to-end).
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.model_metadata import is_local_endpoint
|
||||
|
||||
|
||||
def _resolve_stale_timeout(base_url, est_tokens, stale_base=180.0):
|
||||
"""Mirror of the stale-stream detector resolution."""
|
||||
if stale_base == 180.0 and base_url and is_local_endpoint(base_url):
|
||||
return float("inf") # detector disabled for local providers
|
||||
if est_tokens > 100_000:
|
||||
return max(stale_base, 300.0)
|
||||
if est_tokens > 50_000:
|
||||
return max(stale_base, 240.0)
|
||||
return stale_base
|
||||
|
||||
|
||||
def _resolve_read_timeout(base_url, stale_timeout, base_timeout=1800.0):
|
||||
"""Mirror of the httpx socket read-timeout builder (cloud branch)."""
|
||||
read_timeout = float(os.getenv("HERMES_STREAM_READ_TIMEOUT", 120.0))
|
||||
if read_timeout == 120.0 and base_url and is_local_endpoint(base_url):
|
||||
read_timeout = base_timeout
|
||||
elif (
|
||||
read_timeout == 120.0
|
||||
and stale_timeout is not None
|
||||
and stale_timeout != float("inf")
|
||||
and stale_timeout > read_timeout
|
||||
):
|
||||
read_timeout = stale_timeout
|
||||
return read_timeout
|
||||
|
||||
|
||||
CLOUD_URLS = [
|
||||
"https://api.githubcopilot.com",
|
||||
"https://api.openai.com",
|
||||
"https://openrouter.ai/api",
|
||||
"https://api.anthropic.com",
|
||||
]
|
||||
|
||||
|
||||
class TestCloudReadTimeoutFloor:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_env(self):
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mp.delenv("HERMES_STREAM_READ_TIMEOUT", raising=False)
|
||||
yield
|
||||
|
||||
@pytest.mark.parametrize("base_url", CLOUD_URLS)
|
||||
@pytest.mark.parametrize("est_tokens", [0, 10_000, 60_000, 150_000])
|
||||
def test_read_timeout_never_below_stale(self, base_url, est_tokens):
|
||||
"""Core invariant: the socket read timeout >= the stale detector."""
|
||||
stale = _resolve_stale_timeout(base_url, est_tokens)
|
||||
read = _resolve_read_timeout(base_url, stale)
|
||||
assert read >= stale
|
||||
|
||||
@pytest.mark.parametrize("base_url", CLOUD_URLS)
|
||||
def test_small_context_floored_to_stale_base(self, base_url):
|
||||
"""Reported case: ~120s timeouts on Copilot are raised to the 180s base."""
|
||||
stale = _resolve_stale_timeout(base_url, est_tokens=37_000)
|
||||
read = _resolve_read_timeout(base_url, stale)
|
||||
assert read == 180.0
|
||||
|
||||
@pytest.mark.parametrize("base_url", CLOUD_URLS)
|
||||
def test_large_context_tracks_scaled_stale(self, base_url):
|
||||
"""Big contexts scale the stale detector; the read timeout follows."""
|
||||
assert _resolve_read_timeout(base_url, _resolve_stale_timeout(base_url, 60_000)) == 240.0
|
||||
assert _resolve_read_timeout(base_url, _resolve_stale_timeout(base_url, 150_000)) == 300.0
|
||||
|
||||
def test_user_override_is_respected(self):
|
||||
"""An explicit HERMES_STREAM_READ_TIMEOUT is never overridden by the floor."""
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mp.setenv("HERMES_STREAM_READ_TIMEOUT", "90")
|
||||
stale = _resolve_stale_timeout("https://api.githubcopilot.com", est_tokens=0)
|
||||
assert _resolve_read_timeout("https://api.githubcopilot.com", stale) == 90.0
|
||||
|
||||
|
||||
class TestLocalUnaffected:
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_env(self):
|
||||
with pytest.MonkeyPatch.context() as mp:
|
||||
mp.delenv("HERMES_STREAM_READ_TIMEOUT", raising=False)
|
||||
yield
|
||||
|
||||
def test_local_still_raised_to_base(self):
|
||||
"""Local providers keep their existing behavior (raise to base timeout)."""
|
||||
stale = _resolve_stale_timeout("http://localhost:11434", est_tokens=0)
|
||||
assert stale == float("inf") # detector disabled for local
|
||||
read = _resolve_read_timeout("http://localhost:11434", stale)
|
||||
assert read == 1800.0 # not clamped by inf
|
||||
|
||||
def test_stale_none_falls_back_to_default(self):
|
||||
"""If the stale value is unresolved, the read timeout keeps its default."""
|
||||
assert _resolve_read_timeout("https://api.githubcopilot.com", None) == 120.0
|
||||
|
|
@ -55,3 +55,44 @@ class TestContextFileCwd:
|
|||
def test_configured_dir_when_terminal_cwd_set(self, monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
|
||||
assert _captured_context_cwd(_make_agent()) == tmp_path
|
||||
|
||||
|
||||
def _stable_prompt(agent):
|
||||
with (
|
||||
patch("run_agent.load_soul_md", return_value=""),
|
||||
patch("run_agent.build_nous_subscription_prompt", return_value=""),
|
||||
patch("run_agent.build_environment_hints", return_value=""),
|
||||
patch("run_agent.build_context_files_prompt", return_value=""),
|
||||
):
|
||||
return build_system_prompt_parts(agent)["stable"]
|
||||
|
||||
|
||||
class TestCodingContextBlock:
|
||||
def test_injected_when_active(self, monkeypatch, tmp_path):
|
||||
import subprocess
|
||||
|
||||
subprocess.run(["git", "-C", str(tmp_path), "init", "-q"], check=True)
|
||||
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
|
||||
agent = _make_agent(valid_tool_names=["read_file"], platform="cli")
|
||||
stable = _stable_prompt(agent)
|
||||
assert "coding agent" in stable
|
||||
assert "Workspace" in stable
|
||||
|
||||
def test_absent_when_off(self, monkeypatch, tmp_path):
|
||||
import subprocess
|
||||
|
||||
subprocess.run(["git", "-C", str(tmp_path), "init", "-q"], check=True)
|
||||
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
|
||||
agent = _make_agent(valid_tool_names=["read_file"], platform="cli")
|
||||
# Drive the real path: force the resolved mode to "off" via config.
|
||||
with patch("agent.coding_context._coding_mode", return_value="off"):
|
||||
stable = _stable_prompt(agent)
|
||||
assert "coding agent" not in stable
|
||||
|
||||
def test_absent_without_tools(self, monkeypatch, tmp_path):
|
||||
import subprocess
|
||||
|
||||
subprocess.run(["git", "-C", str(tmp_path), "init", "-q"], check=True)
|
||||
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
|
||||
agent = _make_agent(valid_tool_names=[], platform="cli")
|
||||
assert "coding agent" not in _stable_prompt(agent)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue