KrustyPlanet/hermes-agent
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Merge commit '6110aed9b' into feat/whatsapp-cloud-api

Browse source
This commit is contained in:
emozilla 2026-06-10 21:39:22 -04:00
commit bfcc9f92b4
3038 changed files with 499128 additions and 63841 deletions
Download patch file Download diff file Expand all files Collapse all files

3
tests/acp/test_approval_isolation.py
View file

@ -13,11 +13,8 @@ Both fixed together by:
threads don't collide.
"""
import os
import threading
from unittest.mock import MagicMock
import pytest
class TestThreadLocalApprovalCallback:

2
tests/acp/test_events.py
View file

@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import acp
from acp.schema import AgentPlanUpdate, ToolCallStart, ToolCallProgress, AgentThoughtChunk, AgentMessageChunk
from acp.schema import AgentPlanUpdate
from acp_adapter.events import (
_build_plan_update_from_todo_result,

3
tests/acp/test_mcp_e2e.py
View file

@ -7,9 +7,6 @@ Exercises the full flow through the ACP server layer:
session_update events arrive at the mock client
"""
import asyncio
from collections import deque
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest

155
tests/acp/test_server.py
View file

@ -18,7 +18,6 @@ from acp.schema import (
AvailableCommandsUpdate,
Implementation,
InitializeResponse,
ListSessionsResponse,
LoadSessionResponse,
NewSessionResponse,
PromptResponse,
@ -33,7 +32,6 @@ from acp.schema import (
TextContentBlock,
ToolCallProgress,
ToolCallStart,
Usage,
UsageUpdate,
UserMessageChunk,
)
@ -971,6 +969,18 @@ class TestSessionConfiguration:
"hermes_cli.runtime_provider.resolve_runtime_provider",
fake_resolve_runtime_provider,
)
# Pin the parser so this test doesn't depend on live
# ``_KNOWN_PROVIDER_NAMES`` / ``_PROVIDER_ALIASES`` module state
# (sibling of the same hardening on
# ``test_model_switch_uses_requested_provider``).
monkeypatch.setattr(
"hermes_cli.models.parse_model_input",
lambda raw, current: ("anthropic", "claude-sonnet-4-6"),
)
monkeypatch.setattr(
"hermes_cli.models.detect_provider_for_model",
lambda model, current: None,
)
manager = SessionManager(db=SessionDB(tmp_path / "state.db"))
with patch("run_agent.AIAgent", side_effect=fake_agent):
@ -1090,6 +1100,82 @@ class TestPrompt:
]
assert any(update.session_update == "agent_message_chunk" for update in updates)
@pytest.mark.asyncio
async def test_prompt_suppresses_cancel_interrupt_sentinel(self, agent):
"""ACP cancel status text should not be emitted as assistant output."""
new_resp = await agent.new_session(cwd=".")
state = agent.session_manager.get_session(new_resp.session_id)
sentinel = "Operation interrupted: waiting for model response (3.3s elapsed)."
def mock_run(*args, **kwargs):
state.cancel_event.set()
return {
"final_response": sentinel,
"messages": list(state.history),
"interrupted": True,
"completed": False,
}
state.agent.run_conversation = mock_run
mock_conn = MagicMock(spec=acp.Client)
mock_conn.session_update = AsyncMock()
agent._conn = mock_conn
with patch("agent.title_generator.maybe_auto_title") as mock_title:
prompt = [TextContentBlock(type="text", text="please do a long task")]
resp = await agent.prompt(prompt=prompt, session_id=new_resp.session_id)
updates = [
call.kwargs.get("update") or call.args[1]
for call in mock_conn.session_update.call_args_list
]
agent_texts = [
update.content.text
for update in updates
if update.session_update == "agent_message_chunk"
]
assert resp.stop_reason == "cancelled"
assert sentinel not in agent_texts
assert not any(text.startswith("Operation interrupted:") for text in agent_texts)
mock_title.assert_not_called()
@pytest.mark.asyncio
async def test_prompt_keeps_real_final_response_on_cancelled_turn(self, agent):
"""A cancel flag must not suppress actual assistant/model text."""
new_resp = await agent.new_session(cwd=".")
state = agent.session_manager.get_session(new_resp.session_id)
final_text = "The actual model answer arrived before cancellation settled."
def mock_run(*args, **kwargs):
state.cancel_event.set()
return {
"final_response": final_text,
"messages": [],
"interrupted": True,
}
state.agent.run_conversation = mock_run
mock_conn = MagicMock(spec=acp.Client)
mock_conn.session_update = AsyncMock()
agent._conn = mock_conn
prompt = [TextContentBlock(type="text", text="finish if you can")]
resp = await agent.prompt(prompt=prompt, session_id=new_resp.session_id)
updates = [
call.kwargs.get("update") or call.args[1]
for call in mock_conn.session_update.call_args_list
]
agent_texts = [
update.content.text
for update in updates
if update.session_update == "agent_message_chunk"
]
assert resp.stop_reason == "cancelled"
assert final_text in agent_texts
@pytest.mark.asyncio
async def test_prompt_propagates_hermes_session_id_env(self, agent, monkeypatch):
"""ACP must propagate the originating session id to the agent loop
@ -1191,6 +1277,48 @@ class TestPrompt:
assert len(agent_chunks) == 1
assert agent_chunks[0].content.text == "streamed answer"
@pytest.mark.asyncio
async def test_prompt_delivers_transformed_response_after_streaming(self, agent):
"""If a transform_llm_output plugin hook modifies the response after
streaming, ACP must deliver the transformed final_response so the
appended/rewritten text reaches the client.
"""
new_resp = await agent.new_session(cwd=".")
state = agent.session_manager.get_session(new_resp.session_id)
def mock_run(*args, **kwargs):
state.agent.stream_delta_callback("original answer")
return {
"final_response": "original answer\n\n[plugin appended this]",
"response_transformed": True,
"messages": [],
}
state.agent.run_conversation = mock_run
mock_conn = MagicMock(spec=acp.Client)
mock_conn.session_update = AsyncMock()
agent._conn = mock_conn
prompt = [TextContentBlock(type="text", text="hello")]
await agent.prompt(prompt=prompt, session_id=new_resp.session_id)
updates = [
call.kwargs.get("update") or call.args[1]
for call in mock_conn.session_update.call_args_list
]
# The streamed chunk and the post-stream transformed message should
# both be present (final delivery is a separate update_agent_message_text
# call carrying the full transformed text).
all_texts = [
getattr(getattr(u, "content", None), "text", None)
for u in updates
]
assert any(
text and "[plugin appended this]" in text for text in all_texts
), f"expected transformed final to be delivered, got: {all_texts!r}"
@pytest.mark.asyncio
async def test_prompt_auto_titles_session(self, agent):
new_resp = await agent.new_session(cwd=".")
@ -1543,6 +1671,20 @@ class TestSlashCommands:
"hermes_cli.runtime_provider.resolve_runtime_provider",
fake_resolve_runtime_provider,
)
# Pin the model-string parser independently of the live
# ``_KNOWN_PROVIDER_NAMES`` / ``_PROVIDER_ALIASES`` module state.
# Otherwise any test in the same xdist worker that mutates those
# globals (e.g. registers a custom provider that shadows
# ``anthropic``) flakes this one — observed once in CI as
# ``'custom' == 'anthropic'``.
monkeypatch.setattr(
"hermes_cli.models.parse_model_input",
lambda raw, current: ("anthropic", "claude-sonnet-4-6"),
)
monkeypatch.setattr(
"hermes_cli.models.detect_provider_for_model",
lambda model, current: None,
)
manager = SessionManager(db=SessionDB(tmp_path / "state.db"))
with patch("run_agent.AIAgent", side_effect=fake_agent):
@ -1553,7 +1695,14 @@ class TestSlashCommands:
assert "Provider: anthropic" in result
assert state.agent.provider == "anthropic"
assert state.agent.base_url == "https://anthropic.example/v1"
assert runtime_calls[-1] == "anthropic"
# ``state.agent.provider == "anthropic"`` plus the base_url check above
# already prove ``fake_resolve_runtime_provider`` was called with
# ``requested="anthropic"`` for the model-switch step — the agent's
# provider/base_url come from that fake's return value. The legacy
# ``runtime_calls[-1] == "anthropic"`` assertion was flaky in CI
# under specific xdist-slice scheduling (saw ``'custom' == 'anthropic'``
# repeatedly) and was redundant with those checks, so it's gone.
assert "anthropic" in runtime_calls
# ---------------------------------------------------------------------------

201
tests/acp/test_session_db_private_access.py Normal file
View file

@ -0,0 +1,201 @@
"""Tests for the update_session_meta fix.
Verifies that:
1. SessionDB.update_session_meta() exists and works correctly via the
public _execute_write path (not db._lock / db._conn directly).
2. session.py _persist() no longer touches db._lock or db._conn.
3. update_session_meta updates the correct columns atomically.
"""
import ast
import json
import tempfile
from pathlib import Path
from unittest.mock import MagicMock, patch, call
import pytest
from hermes_state import SessionDB
from acp_adapter.session import SessionManager
def _tmp_db(tmp_path):
return SessionDB(db_path=tmp_path / "state.db")
def _mock_agent():
return MagicMock(name="MockAIAgent")
# ---------------------------------------------------------------------------
# hermes_state.SessionDB.update_session_meta — unit tests
# ---------------------------------------------------------------------------
class TestUpdateSessionMeta:
"""Direct unit tests for the new public method."""
def test_method_exists(self, tmp_path):
db = _tmp_db(tmp_path)
assert hasattr(db, "update_session_meta"), (
"SessionDB must have update_session_meta() public method"
)
assert callable(db.update_session_meta)
def test_updates_model_config(self, tmp_path):
db = _tmp_db(tmp_path)
db.create_session("s1", source="acp", model="gpt-4")
new_meta = json.dumps({"cwd": "/new/path", "provider": "openai"})
db.update_session_meta("s1", new_meta, model=None)
row = db.get_session("s1")
stored = json.loads(row["model_config"])
assert stored["cwd"] == "/new/path"
assert stored["provider"] == "openai"
def test_updates_model_when_provided(self, tmp_path):
db = _tmp_db(tmp_path)
db.create_session("s2", source="acp", model="gpt-3.5")
db.update_session_meta("s2", json.dumps({"cwd": "."}), model="gpt-4o")
row = db.get_session("s2")
assert row["model"] == "gpt-4o"
def test_preserves_existing_model_when_none(self, tmp_path):
"""Passing model=None must leave the stored model unchanged (COALESCE)."""
db = _tmp_db(tmp_path)
db.create_session("s3", source="acp", model="claude-3")
db.update_session_meta("s3", json.dumps({"cwd": "."}), model=None)
row = db.get_session("s3")
assert row["model"] == "claude-3"
def test_uses_execute_write_not_private_api(self, tmp_path):
"""update_session_meta must route through _execute_write, not _conn directly."""
db = _tmp_db(tmp_path)
db.create_session("s4", source="acp")
call_count = [0]
original = db._execute_write
def patched(fn):
call_count[0] += 1
return original(fn)
db._execute_write = patched
db.update_session_meta("s4", json.dumps({"cwd": "."}), model="m")
assert call_count[0] >= 1, (
"update_session_meta must call _execute_write at least once"
)
def test_noop_on_nonexistent_session(self, tmp_path):
"""Updating a non-existent session must not raise."""
db = _tmp_db(tmp_path)
db.update_session_meta("ghost", json.dumps({"cwd": "."}), model=None)
# ---------------------------------------------------------------------------
# AST check: session.py must not access db._lock or db._conn
# ---------------------------------------------------------------------------
class TestNoPrviateDBAccess:
"""_persist() in session.py must not access db._lock or db._conn."""
def test_no_db_private_lock_access(self):
with open("acp_adapter/session.py", encoding="utf-8") as f:
source = f.read()
tree = ast.parse(source)
violations = []
for node in ast.walk(tree):
# Looking for: db._lock or db._conn
if isinstance(node, ast.Attribute):
if isinstance(node.value, ast.Name) and node.value.id == "db":
if node.attr in ("_lock", "_conn"):
violations.append(
f"db.{node.attr} at line {node.lineno}"
)
assert violations == [], (
"session.py accesses private SessionDB internals: "
+ ", ".join(violations)
+ " — use db.update_session_meta() instead"
)
def test_persist_calls_update_session_meta(self):
"""AST check: _persist must call db.update_session_meta()."""
with open("acp_adapter/session.py", encoding="utf-8") as f:
tree = ast.parse(f.read())
found = False
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name == "_persist":
for child in ast.walk(node):
if isinstance(child, ast.Call):
func = child.func
if isinstance(func, ast.Attribute):
if func.attr == "update_session_meta":
found = True
break
break
assert found, (
"_persist() must call db.update_session_meta() "
"instead of db._conn.execute() directly"
)
# ---------------------------------------------------------------------------
# Integration: _persist round-trip via SessionManager
# ---------------------------------------------------------------------------
class TestPersistRoundTrip:
"""End-to-end: save a session and verify DB state is correct."""
def test_cwd_persisted_via_update_session_meta(self, tmp_path):
db = _tmp_db(tmp_path)
manager = SessionManager(agent_factory=_mock_agent, db=db)
state = manager.create_session(cwd="/original")
assert db.get_session(state.session_id) is not None
# Simulate cwd change and save
state.cwd = "/updated"
manager.save_session(state.session_id)
row = db.get_session(state.session_id)
mc = json.loads(row["model_config"])
assert mc["cwd"] == "/updated"
def test_model_persisted_via_update_session_meta(self, tmp_path):
db = _tmp_db(tmp_path)
manager = SessionManager(agent_factory=_mock_agent, db=db)
state = manager.create_session()
state.model = "new-model-xyz"
manager.save_session(state.session_id)
row = db.get_session(state.session_id)
assert row["model"] == "new-model-xyz"
def test_existing_model_not_cleared_on_save(self, tmp_path):
"""If state.model is empty, the DB model column must not be overwritten."""
db = _tmp_db(tmp_path)
manager = SessionManager(agent_factory=_mock_agent, db=db)
state = manager.create_session()
# Manually set a model in DB
db.update_session_meta(state.session_id, json.dumps({"cwd": "."}), model="stored-model")
# Now save with empty model
state.model = ""
manager.save_session(state.session_id)
row = db.get_session(state.session_id)
assert row["model"] == "stored-model", (
"COALESCE must preserve the existing model when new value is NULL"
)

103
tests/acp/test_session_provenance.py Normal file
View file

@ -0,0 +1,103 @@
"""Tests for ACP session-provenance derivation (issue #33617).
Exercises acp_adapter.provenance against a real SessionDB — no mocks — covering
the acceptance-criteria matrix: root session, compression-split continuation,
multi-depth chains, rotation flagging, and graceful handling of unknown ids.
"""
import time
import pytest
from acp_adapter.provenance import build_session_provenance, session_provenance_meta
from hermes_state import SessionDB
@pytest.fixture()
def db(tmp_path):
d = SessionDB(db_path=tmp_path / "state.db")
yield d
def _mk(db, sid, parent=None):
db.create_session(session_id=sid, source="acp", parent_session_id=parent)
def test_root_session_no_compression(db):
_mk(db, "root1")
prov = build_session_provenance(db, "acp-1", "root1")
assert prov["acpSessionId"] == "acp-1"
assert prov["currentHermesSessionId"] == "root1"
assert prov["rootHermesSessionId"] == "root1"
assert prov["parentHermesSessionId"] is None
assert prov["sessionKind"] == "root"
assert prov["compressionDepth"] == 0
assert "reason" not in prov # no rotation signalled
def test_compression_split_continuation(db):
# Parent ended with compression, child created afterwards.
_mk(db, "old")
db.end_session("old", "compression")
time.sleep(0.001)
_mk(db, "new", parent="old")
prov = build_session_provenance(
db, "acp-1", "new", previous_hermes_session_id="old"
)
assert prov["sessionKind"] == "continuation"
assert prov["parentHermesSessionId"] == "old"
assert prov["rootHermesSessionId"] == "old"
assert prov["compressionDepth"] == 1
assert prov["previousHermesSessionId"] == "old"
# Head rotated this turn → reason/creatorKind flagged.
assert prov["reason"] == "compression"
assert prov["creatorKind"] == "compression"
def test_multi_depth_chain(db):
_mk(db, "s0")
db.end_session("s0", "compression")
_mk(db, "s1", parent="s0")
db.end_session("s1", "compression")
_mk(db, "s2", parent="s1")
prov = build_session_provenance(db, "acp-1", "s2")
assert prov["rootHermesSessionId"] == "s0"
assert prov["compressionDepth"] == 2
assert prov["sessionKind"] == "continuation"
def test_non_compression_parent_is_root_not_continuation(db):
# A child with a parent that did NOT end via compression (e.g. delegate
# or branch child) must not be reported as a compression continuation.
_mk(db, "p")
_mk(db, "c", parent="p") # parent still live, no end_reason
prov = build_session_provenance(db, "acp-1", "c")
assert prov["sessionKind"] == "root"
assert prov["compressionDepth"] == 0
assert prov["rootHermesSessionId"] == "p" # lineage root still walked
def test_no_false_rotation_when_head_unchanged(db):
_mk(db, "s")
# previous == current → no rotation reason emitted.
prov = build_session_provenance(
db, "acp-1", "s", previous_hermes_session_id="s"
)
assert "reason" not in prov
assert "creatorKind" not in prov
assert prov["previousHermesSessionId"] == "s"
def test_unknown_session_returns_none(db):
assert build_session_provenance(db, "acp-1", "does-not-exist") is None
assert session_provenance_meta(db, "acp-1", "does-not-exist") is None
def test_meta_wrapper_shape(db):
_mk(db, "root1")
meta = session_provenance_meta(db, "acp-1", "root1")
assert set(meta.keys()) == {"hermes"}
assert "sessionProvenance" in meta["hermes"]
assert meta["hermes"]["sessionProvenance"]["currentHermesSessionId"] == "root1"

1
tests/acp/test_tools.py
View file

@ -1,6 +1,5 @@
"""Tests for acp_adapter.tools — tool kind mapping and ACP content building."""
import pytest
from acp_adapter.edit_approval import EditProposal
from acp_adapter.tools import (

2
tests/agent/lsp/test_backend_gate.py
View file

@ -8,8 +8,6 @@ syntax check exactly as if LSP were disabled.
"""
from __future__ import annotations
import os
import sys
from unittest.mock import MagicMock
import pytest

5
tests/agent/lsp/test_broken_set.py
View file

@ -14,15 +14,12 @@ This module verifies:
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
from unittest.mock import MagicMock, patch
from unittest.mock import patch
import pytest
from agent.lsp.manager import LSPService
from agent.lsp.servers import SERVERS, ServerContext, ServerDef, SpawnSpec
from agent.lsp.workspace import clear_cache

6
tests/agent/lsp/test_diagnostics_field.py
View file

@ -6,12 +6,8 @@ having LSP output prepended to the lint string.
"""
from __future__ import annotations
import os
import sys
import tempfile
from unittest.mock import MagicMock, patch
from unittest.mock import patch
import pytest
from tools.environments.local import LocalEnvironment
from tools.file_operations import (

41
tests/agent/lsp/test_install_and_lint_fixes.py
View file

@ -94,6 +94,47 @@ def test_install_npm_works_without_extras(tmp_path, monkeypatch):
assert install_targets == ["pyright"]
def test_existing_binary_finds_windows_wrapper_in_staging(tmp_path, monkeypatch):
"""Installed Windows shims should satisfy later status/probe calls."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from agent.lsp import install as install_mod
wrapper = install_mod.hermes_lsp_bin_dir() / "pyright-langserver.cmd"
wrapper.write_text("@echo off\n")
wrapper.chmod(0o755)
monkeypatch.setattr(install_mod, "_is_windows", lambda: True)
monkeypatch.setattr(install_mod.shutil, "which", lambda _name: None)
assert install_mod._existing_binary("pyright-langserver") == str(wrapper)
assert install_mod.detect_status("pyright") == "installed"
def test_install_pip_finds_windows_scripts_launcher(tmp_path, monkeypatch):
"""pip console scripts can land in Scripts/ on native Windows."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
from agent.lsp import install as install_mod
def fake_run(cmd, **kwargs):
scripts_dir = install_mod.hermes_lsp_bin_dir().parent / "python-packages" / "Scripts"
scripts_dir.mkdir(parents=True, exist_ok=True)
launcher = scripts_dir / "fake-language-server.exe"
launcher.write_text("launcher\n")
launcher.chmod(0o755)
return MagicMock(returncode=0, stderr="")
monkeypatch.setattr(install_mod, "_is_windows", lambda: True)
monkeypatch.setattr(install_mod.subprocess, "run", fake_run)
resolved = install_mod._install_pip("fake-lsp", "fake-language-server")
assert resolved is not None
assert resolved.endswith("fake-language-server.exe")
assert (install_mod.hermes_lsp_bin_dir() / "fake-language-server.exe").exists()
# ---------------------------------------------------------------------------
# Fix 2: ``hermes lsp status`` surfaces shellcheck-missing for bash
# ---------------------------------------------------------------------------

2
tests/agent/lsp/test_lifecycle.py
View file

@ -7,7 +7,7 @@ pyright/gopls/etc. are still alive on the host.
from __future__ import annotations
import atexit
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
import pytest

1
tests/agent/lsp/test_reporter.py
View file

@ -2,7 +2,6 @@
from __future__ import annotations
from agent.lsp.reporter import (
DEFAULT_SEVERITIES,
MAX_PER_FILE,
format_diagnostic,
report_for_file,

2
tests/agent/lsp/test_service.py
View file

@ -7,7 +7,6 @@ on.
"""
from __future__ import annotations
import os
import sys
from pathlib import Path
@ -19,7 +18,6 @@ from agent.lsp.servers import (
ServerContext,
ServerDef,
SpawnSpec,
find_server_for_file,
)

172
tests/agent/test_anthropic_adapter.py
View file

@ -1,6 +1,7 @@
"""Tests for agent/anthropic_adapter.py — Anthropic Messages API adapter."""
import json
import sys
import time
from types import SimpleNamespace
from unittest.mock import patch, MagicMock
@ -420,6 +421,24 @@ class TestWriteClaudeCodeCredentials:
assert data["otherField"] == "keep-me"
assert data["claudeAiOauth"]["accessToken"] == "new-tok"
@pytest.mark.skipif(sys.platform.startswith("win"), reason="POSIX mode bits not enforced on Windows")
def test_credentials_file_created_with_0o600(self, tmp_path, monkeypatch):
"""Refreshed Claude Code credentials must land on disk at 0o600.
Regression for the TOCTOU race where ``write_text`` + ``replace``
+ post-write ``chmod`` left both the temp file and the destination
briefly readable at the process umask (commonly 0o644). Mirrors
the fix shipped in #19673 (google_oauth) and #21148 (mcp_oauth).
"""
import stat as _stat
monkeypatch.setattr("agent.anthropic_adapter.Path.home", lambda: tmp_path)
_write_claude_code_credentials("tok", "ref", 12345)
cred_file = tmp_path / ".claude" / ".credentials.json"
assert cred_file.exists()
mode = _stat.S_IMODE(cred_file.stat().st_mode)
assert mode == 0o600, f"creds file mode {oct(mode)} != 0o600 — TOCTOU race regressed"
class TestResolveWithRefresh:
def test_auto_refresh_on_expired_creds(self, monkeypatch, tmp_path):
@ -1169,20 +1188,98 @@ class TestBuildAnthropicKwargs:
# params through its signature, we exercise the strip behavior by
# calling the internal predicate directly.
from agent.anthropic_adapter import _forbids_sampling_params
assert _forbids_sampling_params("claude-opus-4-8") is True
assert _forbids_sampling_params("claude-opus-4-8-fast") is True
assert _forbids_sampling_params("claude-opus-4-7") is True
assert _forbids_sampling_params("claude-opus-4-6") is False
assert _forbids_sampling_params("claude-sonnet-4-5") is False
def test_supports_fast_mode_predicate(self):
"""Fast mode is Opus 4.6 only — Opus 4.7 and others must be excluded."""
"""Fast mode is Opus 4.6 only — Opus 4.7 and others must be excluded.
For Opus 4.8 the fast variant is a separate model ID
(anthropic/claude-opus-4.8-fast) routed through the normal model
field, NOT via the ``speed: "fast"`` request parameter. So
``_supports_fast_mode`` (which gates the parameter) must stay
False for both opus-4-8 and opus-4-8-fast.
"""
from agent.anthropic_adapter import _supports_fast_mode
assert _supports_fast_mode("claude-opus-4-6") is True
assert _supports_fast_mode("anthropic/claude-opus-4-6") is True
assert _supports_fast_mode("claude-opus-4-7") is False
assert _supports_fast_mode("claude-opus-4-8") is False
assert _supports_fast_mode("claude-opus-4-8-fast") is False
assert _supports_fast_mode("claude-sonnet-4-6") is False
assert _supports_fast_mode("claude-haiku-4-5") is False
assert _supports_fast_mode("") is False
def test_fable_class_models_route_as_adaptive_thinking(self):
"""Invariant: unknown/new Claude models default to the modern (4.7+)
contract — adaptive thinking, xhigh-capable, sampling-params-forbidden —
without any per-model code change. Named models (claude-fable-5) and
hypothetical future ones must all classify modern; only the explicit
legacy list stays on the manual path.
"""
from agent.anthropic_adapter import (
_supports_adaptive_thinking,
_supports_xhigh_effort,
_forbids_sampling_params,
_get_anthropic_max_output,
)
# New / unknown Claude models → modern contract by default.
for m in (
"claude-fable-5",
"anthropic/claude-fable-5",
"claude-saga-2", # hypothetical future named model
"anthropic/claude-opus-9", # hypothetical future numbered model
):
assert _supports_adaptive_thinking(m) is True, m
assert _supports_xhigh_effort(m) is True, m
assert _forbids_sampling_params(m) is True, m
# 1M-context reasoning model → highest output ceiling.
assert _get_anthropic_max_output("anthropic/claude-fable-5") == 128_000
def test_legacy_claude_stays_on_manual_thinking(self):
"""Older Claude families keep the legacy manual-thinking contract."""
from agent.anthropic_adapter import (
_supports_adaptive_thinking,
_forbids_sampling_params,
)
for m in (
"claude-3-5-sonnet",
"claude-3-7-sonnet",
"anthropic/claude-opus-4.5",
"anthropic/claude-sonnet-4.5",
"claude-haiku-4-5",
):
assert _supports_adaptive_thinking(m) is False, m
assert _forbids_sampling_params(m) is False, m
def test_claude_46_is_adaptive_but_not_xhigh_or_no_sampling(self):
"""4.6 is adaptive, but predates xhigh and still accepts sampling."""
from agent.anthropic_adapter import (
_supports_adaptive_thinking,
_supports_xhigh_effort,
_forbids_sampling_params,
)
for m in ("claude-opus-4.6", "claude-sonnet-4-6"):
assert _supports_adaptive_thinking(m) is True, m
assert _supports_xhigh_effort(m) is False, m
assert _forbids_sampling_params(m) is False, m
def test_non_claude_anthropic_models_use_manual_path(self):
"""Non-Claude Anthropic-Messages models (minimax, qwen3, kimi) must not
be misclassified as adaptive by the default-to-modern rule."""
from agent.anthropic_adapter import (
_supports_adaptive_thinking,
_supports_xhigh_effort,
_forbids_sampling_params,
)
for m in ("minimax-m2", "qwen3-max", "moonshotai/kimi-k2.5", "glm-4.6"):
assert _supports_adaptive_thinking(m) is False, m
assert _supports_xhigh_effort(m) is False, m
assert _forbids_sampling_params(m) is False, m
def test_fast_mode_omitted_for_unsupported_model(self):
"""fast_mode=True on Opus 4.7 must NOT inject speed=fast (API 400s)."""
kwargs = build_anthropic_kwargs(
@ -1797,6 +1894,79 @@ class TestThinkingBlockSignatureManagement:
assert len(last_thinking) == 1
assert last_thinking[0]["signature"] == "sig_3"
def test_orphan_stripped_tool_use_demotes_dead_signed_thinking(self):
"""Regression: extended-thinking + interrupted parallel tool batch.
An assistant turn with a signed thinking block fires several parallel
tool_use blocks, but the batch is interrupted before every tool_result
comes back. On replay, the orphaned tool_use is stripped — which mutates
the turn and invalidates the thinking-block signature (it was computed
against the original, un-stripped content). Anthropic then rejects the
turn with HTTP 400 "thinking blocks in the latest assistant message
cannot be modified", a non-retryable error that crash-loops the gateway.
The signed thinking block on the mutated latest turn must be demoted to
a plain text block so the turn replays cleanly.
"""
messages = [
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "tc_kept", "function": {"name": "tool_a", "arguments": "{}"}},
{"id": "tc_orphan", "function": {"name": "tool_b", "arguments": "{}"}},
],
"reasoning_details": [
{"type": "thinking", "thinking": "Plan: call A and B.", "signature": "sig_dead"},
],
},
# Only one of the two parallel tool_use blocks got a result back.
{"role": "tool", "tool_call_id": "tc_kept", "content": "result A"},
]
_, result = convert_messages_to_anthropic(messages)
assistant = next(m for m in result if m["role"] == "assistant")
blocks = assistant["content"]
# No signed thinking block survives — the signature is dead.
assert not any(
isinstance(b, dict) and b.get("type") in {"thinking", "redacted_thinking"}
for b in blocks
)
# The reasoning text is preserved as a text block (not silently lost).
text_contents = [b.get("text", "") for b in blocks if b.get("type") == "text"]
assert "Plan: call A and B." in text_contents
# The orphaned tool_use is gone; the answered one survives.
tool_use_ids = [b.get("id") for b in blocks if b.get("type") == "tool_use"]
assert tool_use_ids == ["tc_kept"]
# Internal bookkeeping flag must never leak into the API payload.
assert "_thinking_signature_invalidated" not in assistant
def test_signed_thinking_preserved_when_no_tool_use_stripped(self):
"""Control: an intact latest turn keeps its signed thinking verbatim.
This guards against the orphan-strip fix over-firing — when no tool_use
is removed, the signature is still valid and must be replayed as-is.
"""
messages = [
{
"role": "assistant",
"content": "",
"tool_calls": [
{"id": "tc_1", "function": {"name": "tool_a", "arguments": "{}"}},
],
"reasoning_details": [
{"type": "thinking", "thinking": "Valid plan.", "signature": "sig_live"},
],
},
{"role": "tool", "tool_call_id": "tc_1", "content": "result A"},
]
_, result = convert_messages_to_anthropic(messages)
assistant = next(m for m in result if m["role"] == "assistant")
thinking = [b for b in assistant["content"] if b.get("type") == "thinking"]
assert len(thinking) == 1
assert thinking[0]["signature"] == "sig_live"
assert "_thinking_signature_invalidated" not in assistant
# ---------------------------------------------------------------------------
# Tool choice

2
tests/agent/test_anthropic_keychain.py
View file

@ -1,10 +1,8 @@
"""Tests for Bug #12905 fixes in agent/anthropic_adapter.py — macOS Keychain support."""
import json
import platform
from unittest.mock import patch, MagicMock
import pytest
from agent.anthropic_adapter import (
_read_claude_code_credentials_from_keychain,

94
tests/agent/test_anthropic_kwargs_sanitize.py Normal file
View file

@ -0,0 +1,94 @@
"""Tests for sanitize_anthropic_kwargs (#31673).
Guards the Anthropic Messages dispatch boundary against Responses-API-only
kwargs (``instructions``, ``input``, ``store``, ``parallel_tool_calls``)
leaking in under an api_mode-flip race. The Anthropic SDK raises a
non-retryable ``TypeError`` on any of them, killing the whole turn.
"""
import logging
import pytest
from agent.anthropic_adapter import (
_RESPONSES_ONLY_KWARGS,
sanitize_anthropic_kwargs,
)
def _fake_anthropic_call(**kwargs):
"""Mimic the Anthropic SDK's strict kwarg signature."""
allowed = {
"model", "messages", "max_tokens", "system", "tools", "tool_choice",
"extra_body", "extra_headers", "temperature", "top_p", "top_k",
"thinking", "timeout",
}
bad = set(kwargs) - allowed
if bad:
raise TypeError(
"Messages.stream() got an unexpected keyword argument "
f"{sorted(bad)[0]!r}"
)
return "OK"
def test_bare_leaked_payload_reproduces_the_typeerror():
"""Without the guard, a Responses-shaped payload raises the issue's error."""
with pytest.raises(TypeError, match="unexpected keyword argument"):
_fake_anthropic_call(model="claude-sonnet-4-6", instructions="sys")
def test_strips_all_responses_only_keys():
payload = {
"model": "claude-sonnet-4-6",
"instructions": "You are Hermes.",
"input": [{"role": "user", "content": "hi"}],
"store": False,
"parallel_tool_calls": True,
}
out = sanitize_anthropic_kwargs(payload)
assert out is payload # mutates in place and returns same dict
assert payload == {"model": "claude-sonnet-4-6"}
assert _fake_anthropic_call(**payload) == "OK"
def test_clean_anthropic_payload_is_untouched():
payload = {
"model": "claude-sonnet-4-6",
"messages": [{"role": "user", "content": "hi"}],
"max_tokens": 1024,
"system": "sys",
"tools": [{"name": "x"}],
}
snapshot = dict(payload)
sanitize_anthropic_kwargs(payload)
assert payload == snapshot
assert _fake_anthropic_call(**payload) == "OK"
def test_warns_when_keys_are_stripped(caplog):
with caplog.at_level(logging.WARNING, logger="agent.anthropic_adapter"):
sanitize_anthropic_kwargs(
{"model": "m", "instructions": "sys"}, log_prefix="[pfx] "
)
assert any(
"31673" in r.message and "[pfx] " in r.message
for r in caplog.records
), caplog.records
def test_no_warning_on_clean_payload(caplog):
with caplog.at_level(logging.WARNING, logger="agent.anthropic_adapter"):
sanitize_anthropic_kwargs({"model": "m", "messages": []})
assert not caplog.records
def test_non_dict_input_is_noop():
assert sanitize_anthropic_kwargs(None) is None
assert sanitize_anthropic_kwargs("not a dict") == "not a dict"
def test_responses_only_kwargs_membership():
# Contract: instructions (the reported symptom) plus the sibling
# Responses-shape keys are all covered.
assert {"instructions", "input", "store", "parallel_tool_calls"} <= _RESPONSES_ONLY_KWARGS

248
tests/agent/test_anthropic_mcp_prefix_strip.py Normal file
View file

@ -0,0 +1,248 @@
"""Tests for GH-25255: Anthropic OAuth mcp_ prefix stripping.
When strip_tool_prefix=True (Anthropic OAuth path), the transport must only
strip the ``mcp_`` prefix from OAuth-injected tools, NOT from Hermes-native
MCP server tools that are registered under their full ``mcp_<server>_<tool>``
name in the tool registry.
"""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import patch
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_tool_use_block(name: str, block_id: str = "tc_1", input_data: dict | None = None):
"""Create a fake Anthropic tool_use content block."""
return SimpleNamespace(
type="tool_use",
id=block_id,
name=name,
input=input_data or {"query": "test"},
)
def _make_response(*blocks, stop_reason="end_turn"):
"""Create a fake Anthropic Messages response."""
return SimpleNamespace(
content=list(blocks),
stop_reason=stop_reason,
model="claude-sonnet-4",
usage=SimpleNamespace(input_tokens=100, output_tokens=50),
)
class _FakeRegistry:
"""Minimal fake tool registry for testing prefix stripping logic."""
def __init__(self, registered_names: set[str]):
self._names = registered_names
def get_entry(self, name: str):
if name in self._names:
return SimpleNamespace(name=name) # truthy = tool exists
return None
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
class TestAnthropicMcpPrefixStrip:
"""Verify that strip_tool_prefix only strips OAuth-injected prefixes."""
def _get_transport(self):
from agent.transports.anthropic import AnthropicTransport
return AnthropicTransport()
def test_strips_prefix_for_oauth_injected_tool(self):
"""OAuth tools: mcp_read_file -> read_file (stripped).
The tool was registered as 'read_file' in the registry.
Anthropic sees 'mcp_read_file' because Hermes adds the prefix.
On response, we must strip it back to 'read_file'.
"""
transport = self._get_transport()
block = _make_tool_use_block("mcp_read_file")
response = _make_response(block)
registry = _FakeRegistry({"read_file", "terminal", "web_search"})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "read_file"
def test_preserves_native_mcp_server_tool_name(self):
"""Native MCP tools: mcp_composio_SEARCH -> mcp_composio_SEARCH (kept).
The tool is registered with the full mcp_ prefix in the registry.
Stripping would break registry lookup.
"""
transport = self._get_transport()
block = _make_tool_use_block("mcp_composio_COMPOSIO_SEARCH_TOOLS")
response = _make_response(block)
registry = _FakeRegistry({
"mcp_composio_COMPOSIO_SEARCH_TOOLS",
"mcp_composio_COMPOSIO_GET_TOOL_SCHEMAS",
"read_file",
})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "mcp_composio_COMPOSIO_SEARCH_TOOLS"
def test_no_strip_when_flag_false(self):
"""When strip_tool_prefix=False, names are never modified."""
transport = self._get_transport()
block = _make_tool_use_block("mcp_read_file")
response = _make_response(block)
registry = _FakeRegistry({"read_file"})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=False)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "mcp_read_file"
def test_no_strip_when_not_mcp_prefixed(self):
"""Non-mcp_ names are untouched regardless of strip flag."""
transport = self._get_transport()
block = _make_tool_use_block("web_search")
response = _make_response(block)
registry = _FakeRegistry({"web_search"})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "web_search"
def test_preserves_name_when_neither_in_registry(self):
"""When neither stripped nor full name is in registry, keep full name.
Safety fallback: if we can't determine the type, prefer the full name
since it's what the LLM was told about.
"""
transport = self._get_transport()
block = _make_tool_use_block("mcp_unknown_tool")
response = _make_response(block)
registry = _FakeRegistry({"read_file"}) # neither name registered
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
assert result.tool_calls[0].name == "mcp_unknown_tool"
def test_mixed_tools_same_response(self):
"""Both OAuth and native MCP tools in the same response."""
transport = self._get_transport()
block1 = _make_tool_use_block("mcp_read_file", block_id="tc_1")
block2 = _make_tool_use_block("mcp_composio_SEARCH", block_id="tc_2")
block3 = _make_tool_use_block("mcp_composio_SEARCH", block_id="tc_3") # also registered natively
response = _make_response(block1, block2, block3)
registry = _FakeRegistry({
"read_file", # OAuth-injected
"mcp_composio_SEARCH", # native MCP
})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 3
# OAuth tool: stripped
assert result.tool_calls[0].name == "read_file"
# Native MCP: preserved (both stripped and full are registered, full wins)
assert result.tool_calls[1].name == "mcp_composio_SEARCH"
assert result.tool_calls[2].name == "mcp_composio_SEARCH"
def test_both_stripped_and_full_registered_prefers_full(self):
"""Edge case: both 'foo' and 'mcp_foo' exist in registry.
Keep 'mcp_foo' (the original name) since it's what the LLM requested.
"""
transport = self._get_transport()
block = _make_tool_use_block("mcp_foo")
response = _make_response(block)
registry = _FakeRegistry({"foo", "mcp_foo"})
with patch("tools.registry.registry", registry):
result = transport.normalize_response(response, strip_tool_prefix=True)
assert len(result.tool_calls) == 1
# Both exist — the condition `get_entry(stripped) and not get_entry(name)`
# is False because get_entry(name) IS truthy, so we keep the full name.
assert result.tool_calls[0].name == "mcp_foo"
class TestAnthropicOAuthOutgoingPrefix:
"""Verify the outgoing-side companion fix: build_anthropic_kwargs must not
double-prefix tool names that already start with ``mcp_`` (native MCP server
tools registered as ``mcp_<server>_<tool>``). GH-25255."""
def _build(self, tools, is_oauth=True):
from agent.anthropic_adapter import build_anthropic_kwargs
return build_anthropic_kwargs(
model="claude-sonnet-4-6",
messages=[{"role": "user", "content": "Hi"}],
tools=tools,
max_tokens=4096,
reasoning_config=None,
is_oauth=is_oauth,
)
def test_oauth_adds_prefix_to_bare_tool_name(self):
"""OAuth + bare name → prefix added (existing Claude Code convention)."""
kwargs = self._build([{
"type": "function",
"function": {"name": "read_file", "description": "x", "parameters": {}},
}])
names = [t["name"] for t in kwargs["tools"]]
assert names == ["mcp_read_file"]
def test_oauth_does_not_double_prefix_native_mcp_tool(self):
"""OAuth + already-prefixed native MCP name → left alone."""
kwargs = self._build([{
"type": "function",
"function": {
"name": "mcp_composio_COMPOSIO_SEARCH_TOOLS",
"description": "x",
"parameters": {},
},
}])
names = [t["name"] for t in kwargs["tools"]]
# Must NOT become "mcp_mcp_composio_..." — that breaks the round-trip
# because normalize_response only strips ONE mcp_ prefix.
assert names == ["mcp_composio_COMPOSIO_SEARCH_TOOLS"]
def test_oauth_mixed_native_and_bare_tools(self):
"""Mixed: native MCP preserved, bare names prefixed."""
kwargs = self._build([
{"type": "function", "function": {"name": "read_file",
"description": "x", "parameters": {}}},
{"type": "function", "function": {"name": "mcp_composio_SEARCH",
"description": "y", "parameters": {}}},
{"type": "function", "function": {"name": "terminal",
"description": "z", "parameters": {}}},
])
names = sorted(t["name"] for t in kwargs["tools"])
assert names == ["mcp_composio_SEARCH", "mcp_read_file", "mcp_terminal"]
def test_non_oauth_path_untouched(self):
"""Non-OAuth requests never get the prefix — schemas pass through as-is."""
kwargs = self._build([
{"type": "function", "function": {"name": "read_file",
"description": "x", "parameters": {}}},
{"type": "function", "function": {"name": "mcp_composio_SEARCH",
"description": "y", "parameters": {}}},
], is_oauth=False)
names = sorted(t["name"] for t in kwargs["tools"])
assert names == ["mcp_composio_SEARCH", "read_file"]

8
tests/agent/test_anthropic_oauth_pkce.py
View file

@ -15,7 +15,6 @@ History:
from __future__ import annotations
import io
import json
from typing import Any, Dict
from urllib.parse import parse_qs, urlparse
@ -53,6 +52,13 @@ def _patch_oauth_flow(
return True
monkeypatch.setattr("webbrowser.open", fake_open)
# The flow now gates webbrowser.open() behind a graphical-browser check so
# it never launches a console browser (w3m/lynx) inside the terminal. Tests
# run headless, so force the GUI path to True — the URL capture relies on
# webbrowser.open() being invoked.
monkeypatch.setattr(
"hermes_cli.auth._can_open_graphical_browser", lambda: True
)
monkeypatch.setattr("builtins.input", lambda *_a, **_kw: callback_code)
class _FakeResponse:

83
tests/agent/test_arcee_trinity_overrides.py
View file

@ -17,6 +17,7 @@ from agent.auxiliary_client import (
_compression_threshold_for_model,
_fixed_temperature_for_model,
_is_arcee_trinity_thinking,
_is_codex_gpt55,
)
@ -74,3 +75,85 @@ def test_compression_threshold_default_none_for_other_models() -> None:
assert _compression_threshold_for_model("trinity-large-preview") is None
assert _compression_threshold_for_model("claude-sonnet-4.6") is None
assert _compression_threshold_for_model("kimi-k2") is None
# ---------------------------------------------------------------------------
# Codex gpt-5.5 compaction-threshold autoraise
#
# ChatGPT's Codex OAuth backend caps gpt-5.5 at a 272K window (verified live:
# ~330K-token request rejected with context_length_exceeded, ~250K accepted).
# The default 50% compaction trigger would fire at ~136K — half the usable
# window — so this route raises the trigger to 85%. Only the Codex OAuth route
# is affected; the same slug on OpenAI direct / OpenRouter / Copilot exposes a
# larger window and keeps the user's global threshold.
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
"model",
[
"gpt-5.5",
"gpt-5.5-pro",
"gpt-5.5-2026-04-23", # dated snapshot
"gpt-5.5-codex-mini", # Codex variant of the 5.5 family (also 272K-capped)
"openai/gpt-5.5", # aggregator-prefixed (still on the codex route)
"GPT-5.5", # case-insensitive
" gpt-5.5 ", # whitespace tolerant
],
)
def test_is_codex_gpt55_matches_on_codex_provider(model: str) -> None:
assert _is_codex_gpt55(model, "openai-codex") is True
@pytest.mark.parametrize(
"provider",
["openrouter", "openai", "copilot", "openai-api", "", None],
)
def test_is_codex_gpt55_rejects_non_codex_providers(provider) -> None:
# gpt-5.5 on any non-Codex route keeps the larger window — no override.
assert _is_codex_gpt55("gpt-5.5", provider) is False
@pytest.mark.parametrize(
"model",
["gpt-5.4", "gpt-5", "gpt-5.55", "gpt-5.50", "", None],
)
def test_is_codex_gpt55_rejects_non_55_models(model) -> None:
# gpt-5.55 / gpt-5.50 are different families and must NOT match — the
# "gpt-5.5-" / "gpt-5.5." prefix guards require a separator after "5.5".
assert _is_codex_gpt55(model, "openai-codex") is False
def test_compression_threshold_for_codex_gpt55() -> None:
assert _compression_threshold_for_model("gpt-5.5", "openai-codex") == 0.85
assert _compression_threshold_for_model("gpt-5.5-pro", "openai-codex") == 0.85
assert _compression_threshold_for_model("openai/gpt-5.5", "openai-codex") == 0.85
def test_compression_threshold_codex_gpt55_other_routes_unaffected() -> None:
# Same slug, different route → no override (keep the user's config value).
assert _compression_threshold_for_model("gpt-5.5", "openrouter") is None
assert _compression_threshold_for_model("gpt-5.5", "openai") is None
assert _compression_threshold_for_model("gpt-5.5", "copilot") is None
assert _compression_threshold_for_model("openai/gpt-5.5") is None # no provider
def test_compression_threshold_codex_gpt55_opt_out() -> None:
# allow_codex_gpt55_autoraise=False reverts to the global default (None).
assert (
_compression_threshold_for_model(
"gpt-5.5", "openai-codex", allow_codex_gpt55_autoraise=False
)
is None
)
def test_compression_threshold_opt_out_does_not_disable_trinity() -> None:
# The opt-out flag is scoped to the Codex gpt-5.5 autoraise; the Arcee
# Trinity override must still apply when the flag is False.
assert (
_compression_threshold_for_model(
"trinity-large-thinking", "openrouter", allow_codex_gpt55_autoraise=False
)
== 0.75
)

1
tests/agent/test_async_utils.py
View file

@ -8,7 +8,6 @@ import warnings
from concurrent.futures import Future
from unittest.mock import patch
import pytest
from agent.async_utils import safe_schedule_threadsafe

986
tests/agent/test_auxiliary_client.py
View file

File diff suppressed because it is too large Load diff

1
tests/agent/test_auxiliary_client_azure_foundry.py
View file

@ -27,7 +27,6 @@ from __future__ import annotations
import sys
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest

153
tests/agent/test_auxiliary_client_xai_oauth_recovery.py Normal file
View file

@ -0,0 +1,153 @@
"""Tests for xAI OAuth 403 error recovery in auxiliary_client.
xAI returns HTTP 403 (not 401) with "unauthenticated:bad-credentials" when
an OAuth2 access token has expired. These tests verify the three fixes:
1. _is_auth_error detects xAI 403 as an auth failure
2. _recoverable_pool_provider maps api.x.ai to xai-oauth
3. _refresh_provider_credentials includes xai-oauth refresh logic
"""
import pytest
# ── _is_auth_error ──────────────────────────────────────────────────────────
def _import_is_auth_error():
from agent.auxiliary_client import _is_auth_error
return _is_auth_error
class TestIsAuthErrorXaiOauth403:
"""Verify _is_auth_error correctly identifies xAI's 403 bad-credentials."""
@pytest.fixture(autouse=True)
def _import(self):
self.is_auth_error = _import_is_auth_error()
def test_xai_403_bad_credentials_is_auth_error(self):
"""The exact error xAI returns for expired OAuth tokens."""
exc = Exception(
"Error code: 403 - {'code': 'The caller does not have permission "
"to execute the specified operation', 'error': 'The OAuth2 access "
"token could not be validated. [WKE=unauthenticated:bad-credentials]'}"
)
exc.status_code = 403 # openai.PermissionDenied sets this
assert self.is_auth_error(exc) is True
def test_xai_403_bad_credentials_without_status_code(self):
"""Fallback match when status_code attribute is missing."""
exc = Exception(
"Error code: 403 - unauthenticated:bad-credentials"
)
# No status_code attribute — should still match via string pattern
assert self.is_auth_error(exc) is True
def test_generic_403_is_not_auth_error(self):
"""A generic 403 (e.g. rate limit, forbidden) should NOT be treated as auth."""
exc = Exception("Error code: 403 - rate limit exceeded")
exc.status_code = 403
assert self.is_auth_error(exc) is False
def test_401_status_code_is_auth_error(self):
"""Existing 401 detection still works."""
exc = Exception("Unauthorized")
exc.status_code = 401
assert self.is_auth_error(exc) is True
def test_401_string_is_auth_error(self):
"""Existing string-based 401 detection still works."""
exc = Exception("Error code: 401 - Unauthorized")
assert self.is_auth_error(exc) is True
def test_authentication_error_class_is_auth_error(self):
"""Existing AuthenticationError class detection still works."""
exc_type = type("AuthenticationError", (Exception,), {})
exc = exc_type("auth failure")
assert self.is_auth_error(exc) is True
def test_permission_denied_without_bad_credentials_is_not_auth_error(self):
"""403 PermissionDenied without bad-credentials should not be auth."""
exc = Exception("Error code: 403 - Permission denied")
exc.status_code = 403
assert self.is_auth_error(exc) is False
def test_500_is_not_auth_error(self):
"""Server errors are not auth errors."""
exc = Exception("Error code: 500 - Internal server error")
exc.status_code = 500
assert self.is_auth_error(exc) is False
def test_unauthenticated_without_bad_credentials_is_not_auth_error(self):
"""'unauthenticated' alone (without 'bad-credentials') should not match."""
exc = Exception("unauthenticated request")
assert self.is_auth_error(exc) is False
# ── _recoverable_pool_provider ──────────────────────────────────────────────
def _import_recoverable_pool_provider():
from agent.auxiliary_client import _recoverable_pool_provider
return _recoverable_pool_provider
class TestRecoverablePoolProviderXaiOAuth:
"""Verify _recoverable_pool_provider maps api.x.ai to xai-oauth."""
@pytest.fixture(autouse=True)
def _import(self):
self.recover = _import_recoverable_pool_provider()
def test_explicit_xai_oauth_provider(self):
"""Explicit provider name passes through."""
result = self.recover("xai-oauth", None)
assert result == "xai-oauth"
def test_api_x_ai_host_match(self):
"""api.x.ai base URL maps to xai-oauth pool."""
class MockClient:
base_url = "https://api.x.ai/v1/"
result = self.recover("auto", MockClient())
assert result == "xai-oauth"
def test_auto_with_unknown_host_returns_none(self):
"""auto provider with unknown host returns None."""
class MockClient:
base_url = "https://unknown.example.com/v1/"
result = self.recover("auto", MockClient())
assert result is None
# ── _refresh_provider_credentials (structure check) ─────────────────────────
def _import_refresh_provider_credentials():
from agent.auxiliary_client import _refresh_provider_credentials
return _refresh_provider_credentials
class TestRefreshProviderCredentialsXaiOAuth:
"""Verify _refresh_provider_credentials has xai-oauth branch.
Full integration testing requires live OAuth tokens, so we verify
the branch exists and handles the no-credential case gracefully.
"""
@pytest.fixture(autouse=True)
def _import(self):
self.refresh = _import_refresh_provider_credentials()
def test_xai_oauth_no_pool_returns_false(self):
"""When no xai-oauth pool exists, refresh returns False gracefully."""
# This tests that the branch exists and doesn't crash.
# It may return True if the singleton resolver finds tokens,
# or False if neither pool nor singleton has credentials.
# Either way, it should not raise an exception.
result = self.refresh("xai-oauth")
assert isinstance(result, bool)
def test_unknown_provider_returns_false(self):
"""Unknown providers fall through to return False."""
result = self.refresh("unknown-provider-xyz")
assert result is False

33
tests/agent/test_auxiliary_config_bridge.py
View file

@ -4,14 +4,11 @@ are properly mapped to environment variables by both CLI and gateway loaders.
Also tests the vision_tools and browser_tool model override env vars.
"""
import json
import os
import sys
from pathlib import Path
from unittest.mock import patch, MagicMock
import pytest
import yaml
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
@ -198,22 +195,32 @@ class TestGatewayBridgeCodeParity:
"""Verify the gateway/run.py config bridge contains the auxiliary section."""
def test_gateway_has_auxiliary_bridge(self):
"""The gateway config bridge must include auxiliary.* bridging."""
"""The gateway config bridge must include auxiliary.* bridging.
After the plugin-aux-task API refactor (2026-05), gateway env-var
names are derived dynamically (``AUXILIARY_<KEY_UPPER>_*``) so the
literal strings ``AUXILIARY_VISION_PROVIDER`` etc. no longer appear
in source. Assert the dynamic shape and the canonical built-in keys
bridged set instead.
"""
gateway_path = Path(__file__).parent.parent.parent / "gateway" / "run.py"
# Pin encoding to UTF-8: source files in this repo are UTF-8, but
# Path.read_text() defaults to the system locale — which is cp1252
# on most Western Windows installs and crashes as soon as the file
# contains any non-ASCII byte (e.g. an em-dash in a comment).
content = gateway_path.read_text(encoding="utf-8")
# Check for key patterns that indicate the bridge is present
assert "AUXILIARY_VISION_PROVIDER" in content
assert "AUXILIARY_VISION_MODEL" in content
assert "AUXILIARY_VISION_BASE_URL" in content
assert "AUXILIARY_VISION_API_KEY" in content
assert "AUXILIARY_WEB_EXTRACT_PROVIDER" in content
assert "AUXILIARY_WEB_EXTRACT_MODEL" in content
assert "AUXILIARY_WEB_EXTRACT_BASE_URL" in content
assert "AUXILIARY_WEB_EXTRACT_API_KEY" in content
# Dynamic env-var derivation present
assert 'f"AUXILIARY_{_upper}_PROVIDER"' in content
assert 'f"AUXILIARY_{_upper}_MODEL"' in content
assert 'f"AUXILIARY_{_upper}_BASE_URL"' in content
assert 'f"AUXILIARY_{_upper}_API_KEY"' in content
# Built-in bridged keys present
assert "_aux_bridged_keys" in content
assert '"vision"' in content
assert '"web_extract"' in content
assert '"approval"' in content
# Plugin-aux-task discovery hooked into bridging
assert "get_plugin_auxiliary_tasks" in content
def test_gateway_no_compression_env_bridge(self):
"""Gateway should NOT bridge compression config to env vars (config-only)."""

1
tests/agent/test_auxiliary_main_first.py
View file

@ -15,7 +15,6 @@ from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
# ── Text aux tasks — _resolve_auto ──────────────────────────────────────────

1
tests/agent/test_auxiliary_named_custom_providers.py
View file

@ -1,6 +1,5 @@
"""Tests for named custom provider and 'main' alias resolution in auxiliary_client."""
import os
from unittest.mock import patch, MagicMock
import pytest

137
tests/agent/test_auxiliary_user_default_headers.py Normal file
View file

@ -0,0 +1,137 @@
"""Tests for user-configured ``model.default_headers`` in the auxiliary client.
Companion to ``tests/run_agent/test_provider_attribution_headers.py`` (which
covers the main agent client). The main agent turn and the auxiliary client
(title generation, context compression, vision routing) build separate OpenAI
clients, so a ``custom`` endpoint behind a gateway/WAF that rejects the OpenAI
SDK's identifying headers needs the ``model.default_headers`` override applied
on BOTH paths — otherwise the main turn succeeds but auxiliary calls to the
same endpoint still fail with an opaque 4xx/502. (#40033)
"""
from unittest.mock import patch, MagicMock
import pytest
@pytest.fixture(autouse=True)
def _isolate(tmp_path, monkeypatch):
"""Redirect HERMES_HOME so load_config() reads our test config.yaml."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
(hermes_home / "config.yaml").write_text("model:\n default: test-model\n")
def _write_config(tmp_path, config_dict):
import yaml
(tmp_path / ".hermes" / "config.yaml").write_text(yaml.dump(config_dict))
class TestApplyUserDefaultHeadersHelper:
"""Direct unit tests for the merge helper."""
def test_user_headers_merged_and_win(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "m", "default_headers": {"User-Agent": "curl/8.7.1", "X-Extra": "1"}},
})
from agent.auxiliary_client import _apply_user_default_headers
merged = _apply_user_default_headers({"User-Agent": "OpenAI/Python 2.24.0"})
assert merged["User-Agent"] == "curl/8.7.1" # user wins
assert merged["X-Extra"] == "1"
def test_no_config_is_noop_returns_original(self, tmp_path):
_write_config(tmp_path, {"model": {"default": "m"}})
from agent.auxiliary_client import _apply_user_default_headers
original = {"User-Agent": "OpenAI/Python"}
merged = _apply_user_default_headers(original)
assert merged == original
def test_none_headers_with_config_creates_dict(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "m", "default_headers": {"User-Agent": "curl/8.7.1"}},
})
from agent.auxiliary_client import _apply_user_default_headers
merged = _apply_user_default_headers(None)
assert merged == {"User-Agent": "curl/8.7.1"}
def test_none_headers_no_config_returns_none(self, tmp_path):
_write_config(tmp_path, {"model": {"default": "m"}})
from agent.auxiliary_client import _apply_user_default_headers
assert _apply_user_default_headers(None) is None
def test_none_values_skipped(self, tmp_path):
_write_config(tmp_path, {
"model": {"default": "m", "default_headers": {"User-Agent": "curl/8.7.1", "X-Drop": None}},
})
from agent.auxiliary_client import _apply_user_default_headers
merged = _apply_user_default_headers({})
assert merged == {"User-Agent": "curl/8.7.1"}
assert "X-Drop" not in merged
class TestAuxClientHonorsUserDefaultHeaders:
"""Integration: resolve_provider_client must pass overridden headers to OpenAI."""
def test_custom_provider_overrides_sdk_user_agent(self, tmp_path):
"""The #40033 reproduction on the auxiliary path."""
_write_config(tmp_path, {
"model": {
"default": "my-custom-model",
"provider": "custom",
"base_url": "http://localhost:8080/v1",
"default_headers": {"User-Agent": "curl/8.7.1", "X-Extra": "1"},
},
})
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("main", "my-custom-model")
assert client is not None
assert mock_openai.called
headers = mock_openai.call_args.kwargs.get("default_headers", {})
assert headers.get("User-Agent") == "curl/8.7.1"
assert headers.get("X-Extra") == "1"
def test_custom_provider_no_override_sends_no_user_agent(self, tmp_path):
"""Without config, the aux client injects nothing — SDK defaults apply."""
_write_config(tmp_path, {
"model": {
"default": "my-custom-model",
"provider": "custom",
"base_url": "http://localhost:8080/v1",
},
})
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("main", "my-custom-model")
assert client is not None
headers = mock_openai.call_args.kwargs.get("default_headers", {}) or {}
assert "User-Agent" not in headers
def test_named_custom_provider_honors_override(self, tmp_path):
"""A `custom_providers:` entry's aux calls also honor model.default_headers.
This is a distinct construction path (_extra2) from the config-level
`model.provider: custom` path — both must apply the global override.
"""
_write_config(tmp_path, {
"model": {
"default": "test-model",
"default_headers": {"User-Agent": "curl/8.7.1"},
},
"custom_providers": [
{"name": "my-gw", "base_url": "http://my-gw.local/v1", "api_key": "k"},
],
})
with patch("agent.auxiliary_client.OpenAI") as mock_openai:
mock_openai.return_value = MagicMock()
from agent.auxiliary_client import resolve_provider_client
client, model = resolve_provider_client("my-gw", "test-model")
assert client is not None
headers = mock_openai.call_args.kwargs.get("default_headers", {}) or {}
assert headers.get("User-Agent") == "curl/8.7.1"

1
tests/agent/test_azure_identity_adapter.py
View file

@ -23,7 +23,6 @@ import sys
from collections.abc import Callable
from types import SimpleNamespace
from typing import cast
from unittest.mock import MagicMock, patch
import pytest

11
tests/agent/test_bedrock_adapter.py
View file

@ -10,11 +10,9 @@ Covers:
"""
import json
import os
import time
from contextlib import contextmanager
from types import ModuleType, SimpleNamespace
from unittest.mock import MagicMock, patch, PropertyMock
from types import ModuleType
from unittest.mock import MagicMock, patch
import pytest
@ -129,7 +127,7 @@ class TestResolveBedrocRegion:
def test_defaults_to_us_east_1(self):
from agent.bedrock_adapter import resolve_bedrock_region
from unittest.mock import patch, MagicMock
from unittest.mock import MagicMock
mock_session = MagicMock()
mock_session.get_config_variable.return_value = None
with _mock_botocore_session(return_value=mock_session):
@ -137,7 +135,7 @@ class TestResolveBedrocRegion:
def test_falls_back_to_botocore_profile_region(self):
from agent.bedrock_adapter import resolve_bedrock_region
from unittest.mock import patch, MagicMock
from unittest.mock import MagicMock
mock_session = MagicMock()
mock_session.get_config_variable.return_value = "eu-central-1"
with _mock_botocore_session(return_value=mock_session):
@ -145,7 +143,6 @@ class TestResolveBedrocRegion:
def test_botocore_failure_falls_back_to_us_east_1(self):
from agent.bedrock_adapter import resolve_bedrock_region
from unittest.mock import patch
with _mock_botocore_session(side_effect=Exception("no botocore")):
assert resolve_bedrock_region({}) == "us-east-1"

2
tests/agent/test_bedrock_integration.py
View file

@ -9,7 +9,6 @@ Note: Tests that import ``hermes_cli.auth`` or ``hermes_cli.runtime_provider``
require Python 3.10+ due to ``str | None`` type syntax in the import chain.
"""
import os
from unittest.mock import MagicMock, patch
import pytest
@ -93,7 +92,6 @@ class TestResolveProvider:
def test_explicit_bedrock_resolves(self, monkeypatch):
"""When user explicitly requests 'bedrock', it should resolve."""
from hermes_cli.auth import PROVIDER_REGISTRY
# bedrock is in the registry, so resolve_provider should return it
from hermes_cli.auth import resolve_provider
result = resolve_provider("bedrock")

134
tests/agent/test_cascading_interrupt_6600.py Normal file
View file

@ -0,0 +1,134 @@
"""Regression guard for the cascading-interrupt hang (PR #6600).
Original diagnosis and fix by Kristian Vastveit (@kristianvast) in PR #6600,
against the then-inline ``_interruptible_api_call`` /
``_interruptible_streaming_api_call`` methods in run_agent.py. Those methods
have since been extracted into ``agent/chat_completion_helpers.py``, so the
fix is reapplied there and these tests target the extracted functions.
The bug: when ``agent.interrupt()`` fires during an active LLM call, the main
poll loop force-closes the worker-local httpx client to stop token generation.
That raises a transport error (RemoteProtocolError) on the worker — the
EXPECTED consequence of our own close, not a network bug. The streaming retry
loop misclassified it as a transient connection error and retried, each doomed
retry stalling for the full stream-stale timeout (up to 300s). Because the
gateway caches AIAgent instances per session, the stale worker outlived the
turn and raced the next turn's request — the root of the multi-minute
cascading-interrupt hang.
The fix: a request-local ``_request_cancelled`` token set by the poll loop
right before the force-close. The worker's exception handler checks it and
exits cleanly (no retry, no fallback, no "reconnecting" status) instead of
treating the forced error as transient.
"""
import threading
import time
import types
from unittest.mock import MagicMock
import httpx
import pytest
from agent import chat_completion_helpers as cch
class _FakeInterruptError(Exception):
"""Stand-in for the transport error a force-close raises on the worker."""
def _make_agent():
"""A MagicMock agent wired with just enough surface for the helpers."""
agent = MagicMock()
agent.api_mode = "chat_completions"
agent._interrupt_requested = False
agent.verbose_logging = False
# _compute_non_stream_stale_timeout / streaming setup helpers return
# benign values; the real call path is mocked per-test.
agent._compute_non_stream_stale_timeout.return_value = 5.0
return agent
def test_non_streaming_cancel_does_not_surface_network_error():
"""A force-close during a non-streaming call must raise InterruptedError,
not the swallowed transport error."""
agent = _make_agent()
create_calls = {"n": 0}
fake_client = MagicMock()
def _create(**kwargs):
create_calls["n"] += 1
# Simulate the main thread firing an interrupt mid-call, then the
# force-close raising a transport error on this worker.
agent._interrupt_requested = True
time.sleep(0.3) # let the poll loop observe the interrupt + force-close
raise httpx.RemoteProtocolError("peer closed connection")
fake_client.chat.completions.create.side_effect = _create
agent._create_request_openai_client.return_value = fake_client
agent._close_request_openai_client = MagicMock()
agent._abort_request_openai_client = MagicMock()
t0 = time.time()
with pytest.raises(InterruptedError):
cch.interruptible_api_call(agent, {"model": "x", "messages": []})
elapsed = time.time() - t0
# The forced RemoteProtocolError must NOT surface as the raised error.
assert create_calls["n"] == 1
assert elapsed < 3.0, f"interrupt took {elapsed:.1f}s — should be near-instant"
def test_normal_transient_error_still_raises_when_not_cancelled():
"""Regression guard: a real transport error with NO interrupt must still
surface to the caller (so the outer retry loop can recover)."""
agent = _make_agent()
fake_client = MagicMock()
fake_client.chat.completions.create.side_effect = httpx.RemoteProtocolError(
"genuine network drop"
)
agent._create_request_openai_client.return_value = fake_client
agent._close_request_openai_client = MagicMock()
agent._abort_request_openai_client = MagicMock()
agent._interrupt_requested = False
with pytest.raises(httpx.RemoteProtocolError):
cch.interruptible_api_call(agent, {"model": "x", "messages": []})
def test_request_cancelled_token_is_request_local():
"""The cancellation token must be created per call, not shared on the
agent — a stale worker from a previous turn must not see the next turn's
interrupt flag flip back to False and mistake its own forced error for a
network bug. We assert the helper reads agent._interrupt_requested at the
force-close site (request-local token set there), by confirming two
independent calls don't share cancellation state."""
agent = _make_agent()
# First call: interrupted.
fake_client_1 = MagicMock()
def _create_1(**kwargs):
agent._interrupt_requested = True
time.sleep(0.3)
raise httpx.RemoteProtocolError("forced close turn A")
fake_client_1.chat.completions.create.side_effect = _create_1
agent._create_request_openai_client.return_value = fake_client_1
agent._close_request_openai_client = MagicMock()
agent._abort_request_openai_client = MagicMock()
with pytest.raises(InterruptedError):
cch.interruptible_api_call(agent, {"model": "x", "messages": []})
# Second call: NOT interrupted (turn boundary cleared the flag). A genuine
# error must still surface — the previous call's cancellation must not leak.
agent._interrupt_requested = False
fake_client_2 = MagicMock()
fake_client_2.chat.completions.create.side_effect = httpx.RemoteProtocolError(
"genuine drop turn B"
)
agent._create_request_openai_client.return_value = fake_client_2
with pytest.raises(httpx.RemoteProtocolError):
cch.interruptible_api_call(agent, {"model": "x", "messages": []})

1
tests/agent/test_codex_cloudflare_headers.py
View file

@ -29,7 +29,6 @@ import base64
import json
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------

176
tests/agent/test_codex_responses_adapter.py Normal file
View file

@ -0,0 +1,176 @@
from types import SimpleNamespace
import pytest
from agent.codex_responses_adapter import (
_format_responses_error,
_normalize_codex_response,
)
def test_normalize_codex_response_drops_transient_rs_tmp_reasoning_items():
response = SimpleNamespace(
status="completed",
output=[
SimpleNamespace(
type="reasoning",
id="rs_tmp_123",
encrypted_content="opaque-transient",
summary=[],
),
SimpleNamespace(
type="reasoning",
id="rs_456",
encrypted_content="opaque-stable",
summary=[SimpleNamespace(text="stable summary")],
),
SimpleNamespace(
type="message",
role="assistant",
status="completed",
content=[SimpleNamespace(type="output_text", text="done")],
),
],
)
assistant_message, finish_reason = _normalize_codex_response(response)
assert finish_reason == "stop"
assert assistant_message.content == "done"
assert assistant_message.codex_reasoning_items == [
{
"type": "reasoning",
"encrypted_content": "opaque-stable",
"id": "rs_456",
"summary": [{"type": "summary_text", "text": "stable summary"}],
}
]
def test_normalize_codex_response_treats_summary_only_reasoning_as_incomplete():
response = SimpleNamespace(
status="completed",
output=[
SimpleNamespace(
type="reasoning",
id="rs_tmp_789",
encrypted_content="opaque-transient",
summary=[SimpleNamespace(text="still thinking")],
)
],
)
assistant_message, finish_reason = _normalize_codex_response(response)
assert finish_reason == "incomplete"
assert assistant_message.content == ""
assert assistant_message.reasoning == "still thinking"
assert assistant_message.codex_reasoning_items is None
# ---------------------------------------------------------------------------
# _format_responses_error — adapted from anomalyco/opencode#28757.
# Provider failures should surface BOTH the code (rate_limit_exceeded /
# context_length_exceeded / internal_error / server_error) and the message,
# so consumers can tell rate limits apart from context-length failures and
# both apart from generic stream drops.
# ---------------------------------------------------------------------------
def test_format_responses_error_combines_code_and_message():
err = {"code": "rate_limit_exceeded", "message": "Slow down"}
assert _format_responses_error(err, "failed") == "rate_limit_exceeded: Slow down"
def test_format_responses_error_message_only():
err = {"message": "Upstream model unavailable"}
assert _format_responses_error(err, "failed") == "Upstream model unavailable"
def test_format_responses_error_code_only_when_message_empty():
# Some providers/proxies emit a code with an empty message body. We
# used to fall back to ``str(error_obj)`` — a dict dump — which leaked
# ``{'code': 'internal_error', 'message': ''}`` into chat output. Now
# the bare code is surfaced, which is the meaningful field.
err = {"code": "internal_error", "message": ""}
assert _format_responses_error(err, "failed") == "internal_error"
def test_format_responses_error_code_only_when_message_missing():
err = {"code": "server_error"}
assert _format_responses_error(err, "failed") == "server_error"
def test_format_responses_error_attribute_style_payload():
# SDK objects expose ``code``/``message`` as attributes rather than dict
# keys. The helper must accept both shapes since the Responses SDK
# returns SimpleNamespace-style objects on ``response.failed``.
err = SimpleNamespace(code="context_length_exceeded", message="too long")
assert _format_responses_error(err, "failed") == "context_length_exceeded: too long"
def test_format_responses_error_falls_back_to_status_when_empty():
assert (
_format_responses_error(None, "failed")
== "Responses API returned status 'failed'"
)
assert (
_format_responses_error(None, "cancelled")
== "Responses API returned status 'cancelled'"
)
def test_format_responses_error_stringifies_opaque_payload():
# Last-resort: a provider sent something that isn't a dict and has no
# code/message attributes. Surface its repr rather than swallow it
# silently — at least it's visible in logs.
assert _format_responses_error("opaque sentinel", "failed") == "opaque sentinel"
def test_format_responses_error_ignores_non_string_code_message():
# Defensive: a malformed gateway could send numbers/objects in these
# fields. We don't want to crash; we want a best-effort string.
err = {"code": 500, "message": None}
assert _format_responses_error(err, "failed") == "500"
def test_normalize_codex_response_failed_includes_code_in_error():
"""Regression: response_status == 'failed' should surface the error
code, not just the message. Used to leak a bare 'Slow down' string
that was indistinguishable from a generic stream truncation."""
# ``output`` non-empty so we don't trip the "no output items" guard
# before reaching the failed-status branch. Real failed responses
# often DO carry a partial message item alongside the error.
response = SimpleNamespace(
status="failed",
output=[
SimpleNamespace(
type="message",
role="assistant",
status="incomplete",
content=[SimpleNamespace(type="output_text", text="partial")],
),
],
error={"code": "rate_limit_exceeded", "message": "Slow down"},
)
with pytest.raises(RuntimeError, match=r"^rate_limit_exceeded: Slow down$"):
_normalize_codex_response(response)
def test_normalize_codex_response_failed_with_message_only():
"""Backwards-compat: a failed response with only a message field
(no code) should still surface that message verbatim."""
response = SimpleNamespace(
status="failed",
output=[
SimpleNamespace(
type="message",
role="assistant",
status="incomplete",
content=[SimpleNamespace(type="output_text", text="partial")],
),
],
error={"message": "model error"},
)
with pytest.raises(RuntimeError, match=r"^model error$"):
_normalize_codex_response(response)

425
tests/agent/test_codex_ttfb_watchdog.py Normal file
View file

@ -0,0 +1,425 @@
"""Regression tests for the Codex time-to-first-byte (TTFB) watchdog.
The chatgpt.com/backend-api/codex endpoint has an intermittent failure mode
where it accepts the connection but never emits a single stream event. The
watchdog in ``interruptible_api_call`` kills such a connection at a short TTFB
cutoff (instead of waiting out the much longer wall-clock stale timeout) so the
retry loop can reconnect promptly. Once any stream event arrives, the TTFB
watchdog is satisfied and a separate idle watchdog handles streams that stop
emitting SSE events.
The "bytes flowing" signal is ``agent._codex_stream_last_event_ts``, set on
*any* event by ``codex_runtime.run_codex_stream`` — so reasoning-only or
tool-call-only turns (which emit no output-text deltas) are not mistaken for a
stall.
"""
from __future__ import annotations
import sys
import time
import types
from types import SimpleNamespace
import pytest
# Stub optional heavy imports so run_agent imports cleanly in isolation.
sys.modules.setdefault("fire", types.SimpleNamespace(Fire=lambda *a, **k: None))
sys.modules.setdefault("firecrawl", types.SimpleNamespace(Firecrawl=object))
sys.modules.setdefault("fal_client", types.SimpleNamespace())
def _make_codex_agent(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
(tmp_path / "config.yaml").write_text("{}\n", encoding="utf-8")
from run_agent import AIAgent
agent = AIAgent(
model="gpt-5.5",
provider="openai-codex",
api_key="sk-dummy",
base_url="https://chatgpt.com/backend-api/codex",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
platform="cli",
)
# The watchdog is gated on the codex_responses api_mode; assert/force it so
# the test is robust to detection-logic changes elsewhere.
agent.api_mode = "codex_responses"
monkeypatch.setattr(agent, "_emit_status", lambda *a, **k: None)
# Keep the wall-clock stale timeout high so any early kill is unambiguously
# the TTFB path, not the stale-call path.
monkeypatch.setattr(
agent, "_compute_non_stream_stale_timeout", lambda *a, **k: 60.0
)
return agent
def test_ttfb_kills_when_no_stream_event(tmp_path, monkeypatch):
"""Backend accepts the connection but emits no event -> killed at the TTFB
cutoff, well before the 60s wall-clock stale timeout, with a retryable
TimeoutError and a ``codex_ttfb_kill`` close reason."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
stop = {"flag": False}
def fake_hang(api_kwargs, client=None, on_first_delta=None):
# Never set _codex_stream_last_event_ts: simulate zero events arriving.
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
t0 = time.time()
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
elapsed = time.time() - t0
assert "TTFB" in str(excinfo.value)
assert "codex_ttfb_kill" in closes
# ~1s cutoff + 2s join grace; must be far under the 60s stale timeout.
assert elapsed < 15, f"TTFB watchdog took {elapsed:.1f}s"
finally:
stop["flag"] = True
def test_ttfb_default_tolerates_slow_first_event(tmp_path, monkeypatch):
"""With no env var set, the no-byte TTFB default is generous (120s), so a
request whose first stream event is merely slow (~2s of backend admission /
prefill) is NOT killed. This is the subscription-backed Codex case the tight
12s default used to abort mid-prefill."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
# Default behavior: no explicit TTFB override.
monkeypatch.delenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", raising=False)
monkeypatch.delenv("HERMES_CODEX_TTFB_MAX_SECONDS", raising=False)
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
sentinel = SimpleNamespace(ok=True)
def fake_slow_first_event(api_kwargs, client=None, on_first_delta=None):
# Backend is alive but slow to admit: first event lands after ~2s,
# well under the 120s default cutoff. Mark the first byte so the
# no-byte detector sees activity, then return the response.
time.sleep(2.0)
agent._codex_stream_last_event_ts = time.time()
return sentinel
monkeypatch.setattr(agent, "_run_codex_stream", fake_slow_first_event)
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
assert resp is sentinel
assert "codex_ttfb_kill" not in closes
def test_ttfb_includes_silent_hang_hint_for_gpt_5_5(tmp_path, monkeypatch):
"""The no-first-byte watchdog should surface the same actionable hint as the
stale-call timeout path when the model matches the silent-hang heuristic."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
closes: list = []
statuses: list[str] = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(agent, "_buffer_status", lambda msg: statuses.append(msg))
monkeypatch.setattr(agent, "_emit_status", lambda msg: statuses.append(msg))
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
stop = {"flag": False}
def fake_hang(api_kwargs, client=None, on_first_delta=None):
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
message = str(excinfo.value)
assert "gpt-5.4" in message
assert "gpt-5.3-codex" in message
assert "gpt-5.4-codex" in message
assert "codex_ttfb_kill" in closes
assert statuses, "expected a user-facing watchdog status"
assert any("gpt-5.4" in s and "gpt-5.3-codex" in s for s in statuses)
finally:
stop["flag"] = True
def test_ttfb_high_env_is_capped_for_openai_codex(tmp_path, monkeypatch):
"""A stale local env value like 90s must not make openai-codex wait 90s
before reconnecting when the backend emits no SSE frames."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "90")
monkeypatch.setenv("HERMES_CODEX_TTFB_MAX_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
stop = {"flag": False}
def fake_hang(api_kwargs, client=None, on_first_delta=None):
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
t0 = time.time()
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.4", "input": "hi"})
elapsed = time.time() - t0
assert "TTFB threshold: 1s" in str(excinfo.value)
assert "codex_ttfb_kill" in closes
assert elapsed < 15, f"TTFB watchdog ignored cap and took {elapsed:.1f}s"
finally:
stop["flag"] = True
def test_ttfb_does_not_kill_when_events_flow(tmp_path, monkeypatch):
"""Once a stream event has arrived, a generation that runs past the TTFB
cutoff is NOT killed by the watchdog — it completes normally."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
sentinel = SimpleNamespace(ok=True)
def fake_stream(api_kwargs, client=None, on_first_delta=None):
# Bytes flowing: mark stream activity right away, then keep generating
# past the 1s TTFB cutoff before returning a real response.
agent._codex_stream_last_event_ts = time.time()
if on_first_delta:
on_first_delta()
time.sleep(2.0)
return sentinel
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
assert resp is sentinel
assert "codex_ttfb_kill" not in closes
def test_event_idle_kills_after_first_event_then_silence(tmp_path, monkeypatch):
"""If Codex emits an opening SSE event and then goes silent, kill it via
the stream-idle watchdog instead of waiting for the long non-stream stale
timeout."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "10")
monkeypatch.setenv("HERMES_CODEX_EVENT_STALE_TIMEOUT_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent,
"_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent,
"_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
stop = {"flag": False}
def fake_stream(api_kwargs, client=None, on_first_delta=None):
agent._codex_stream_last_event_ts = time.time()
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
assert "after first byte" in str(excinfo.value)
assert "codex_stream_idle_kill" in closes
assert "codex_ttfb_kill" not in closes
finally:
stop["flag"] = True
def test_ttfb_disabled_via_env_zero(tmp_path, monkeypatch):
"""Setting HERMES_CODEX_TTFB_TIMEOUT_SECONDS=0 disables the TTFB watchdog;
a no-event stall then falls through to the (here, 60s) stale timeout, so a
short hang is NOT killed by TTFB."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "0")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
monkeypatch.setattr(
agent, "_close_request_openai_client",
lambda c, reason=None: closes.append(reason),
)
sentinel = SimpleNamespace(ok=True)
def fake_stream(api_kwargs, client=None, on_first_delta=None):
# No event marker, but only briefly — well under the 60s stale timeout.
time.sleep(2.0)
return sentinel
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": "hi"})
assert resp is sentinel
assert "codex_ttfb_kill" not in closes
def test_large_codex_request_waits_instead_of_ttfb_reconnect(tmp_path, monkeypatch):
"""Large Codex inputs can legitimately take longer than the small-request
first-byte cutoff before the first SSE frame. Preserve the full input and
wait instead of killing/retrying at TTFB."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client", lambda c, reason=None: closes.append(reason)
)
monkeypatch.setattr(
agent, "_close_request_openai_client", lambda c, reason=None: closes.append(reason)
)
sentinel = SimpleNamespace(ok=True)
def fake_stream(api_kwargs, client=None, on_first_delta=None):
# No event marker for 2s: this would trip the 1s TTFB watchdog on a
# small request, but should be allowed for a large request.
time.sleep(2.0)
return sentinel
monkeypatch.setattr(agent, "_run_codex_stream", fake_stream)
large_input = "x" * 120_000 # ~30k estimated tokens, above large-request gate.
resp = h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": large_input})
assert resp is sentinel
assert "codex_ttfb_kill" not in closes
def test_large_codex_request_strict_ttfb_env_still_reconnects(tmp_path, monkeypatch):
"""Operators can force the old early-reconnect behavior for large inputs
with HERMES_CODEX_TTFB_STRICT=1."""
from agent import chat_completion_helpers as h
agent = _make_codex_agent(tmp_path, monkeypatch)
monkeypatch.setenv("HERMES_CODEX_TTFB_TIMEOUT_SECONDS", "1")
monkeypatch.setenv("HERMES_CODEX_TTFB_STRICT", "1")
closes: list = []
dummy_client = SimpleNamespace()
monkeypatch.setattr(agent, "_create_request_openai_client", lambda **k: dummy_client)
monkeypatch.setattr(
agent, "_abort_request_openai_client", lambda c, reason=None: closes.append(reason)
)
monkeypatch.setattr(
agent, "_close_request_openai_client", lambda c, reason=None: closes.append(reason)
)
stop = {"flag": False}
def fake_hang(api_kwargs, client=None, on_first_delta=None):
deadline = time.time() + 30
while time.time() < deadline and not stop["flag"] and not agent._interrupt_requested:
time.sleep(0.02)
raise RuntimeError("connection closed")
monkeypatch.setattr(agent, "_run_codex_stream", fake_hang)
large_input = "x" * 120_000
try:
with pytest.raises(TimeoutError) as excinfo:
h.interruptible_api_call(agent, {"model": "gpt-5.5", "input": large_input})
assert "TTFB threshold: 1s" in str(excinfo.value)
assert "codex_ttfb_kill" in closes
finally:
stop["flag"] = True

312
tests/agent/test_compression_concurrent_fork.py Normal file
View file

@ -0,0 +1,312 @@
"""Regression: prevent transcript fork when two paths compress the same session_id.
Damien's incident (Discord, 2026-05-28): a long Hermes session in a Discord
gateway hit the compression threshold at the end of a turn. The parent agent
finished delivering the response and ``conversation_loop.py`` fired
``_spawn_background_review(...)`` — which builds a forked ``AIAgent`` that
inherits ``agent.session_id`` (see ``agent/background_review.py``::
``review_agent.session_id = agent.session_id``). Roughly two seconds later
a synthetic ``Background process proc_… completed`` event arrived and
started a fresh turn on the same parent ``session_id`` (still cached in the
gateway's ``SessionEntry``). Both paths hit preflight compression on the
same parent transcript and called ``_compress_context`` concurrently. Each
ended the parent and created its own CHILD session in ``state.db``, both
parented to the same old id. The gateway's ``SessionEntry`` only caught one
rotation; the other child became an orphan that silently accumulated writes.
Repro shape on Damien's machine:
parent 20260527_234659_e65f0e ended_at=set end_reason='compression'
child 20260528_113619_fc80e1 parent=20260527_234659_e65f0e (in SessionEntry)
child <orphan> parent=20260527_234659_e65f0e (silent writes)
This regression simulates the two concurrent ``compress_context`` calls
against a shared ``state.db`` and asserts that the per-session compression
lock added in this PR prevents the orphan child. Without the lock the
fixture deterministically produces 2 children; with the lock, exactly 1.
"""
from __future__ import annotations
import os
import threading
import time
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from hermes_state import SessionDB
def _build_agent_with_db(db: SessionDB, session_id: str):
"""Build an AIAgent that's wired to ``db`` and pinned to ``session_id``."""
with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
from run_agent import AIAgent
agent = AIAgent(
api_key="test-key",
base_url="https://openrouter.ai/api/v1",
model="test/model",
quiet_mode=True,
session_db=db,
session_id=session_id,
skip_context_files=True,
skip_memory=True,
)
# Stub the compressor so it returns deterministic output and DOESN'T make
# an LLM call. Sleep inside compress() so the two threads' rotations
# actually overlap — without that the OS could happen to serialize them
# and hide the bug.
compressor = MagicMock()
def _compress_with_overlap(*_a, **_kw):
time.sleep(0.25)
return [
{"role": "user", "content": "[CONTEXT COMPACTION] summary"},
{"role": "user", "content": "tail"},
]
compressor.compress.side_effect = _compress_with_overlap
compressor.compression_count = 1
compressor.last_prompt_tokens = 0
compressor.last_completion_tokens = 0
compressor._last_summary_error = None
compressor._last_compress_aborted = False
compressor._last_aux_model_failure_model = None
compressor._last_aux_model_failure_error = None
agent.context_compressor = compressor
return agent
def _count_children(db: SessionDB, parent_sid: str) -> int:
"""Count rows in state.db whose parent_session_id == parent_sid."""
rows = db._conn.execute(
"SELECT id FROM sessions WHERE parent_session_id = ?",
(parent_sid,),
).fetchall()
return len(rows)
def test_concurrent_compression_does_not_fork_session(tmp_path: Path) -> None:
"""Two AIAgents that share a session_id MUST NOT both rotate it.
Without the per-session compression lock this fixture deterministically
produces 2 child sessions (transcript fork). With the lock the second
path aborts cleanly, leaving exactly 1 canonical child.
"""
db = SessionDB(db_path=tmp_path / "state.db")
parent_sid = "PARENT_TEST_SESSION"
db.create_session(parent_sid, source="discord")
# Two agents on the same session_id, both wired to the same db —
# mirrors the parent-turn agent + the background-review fork right
# after a turn ends.
agent_a = _build_agent_with_db(db, parent_sid)
agent_b = _build_agent_with_db(db, parent_sid)
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
def run(agent):
try:
agent._compress_context(messages, "sys", approx_tokens=120_000)
except Exception:
# Surface to the test if either raises — should not happen.
raise
t_a = threading.Thread(target=run, args=(agent_a,), name="main_turn")
t_b = threading.Thread(target=run, args=(agent_b,), name="review_fork")
t_a.start()
t_b.start()
t_a.join(timeout=10)
t_b.join(timeout=10)
# Exactly one canonical child — not two orphans.
assert _count_children(db, parent_sid) == 1, (
"Compression lock failed: parent session has multiple children in state.db "
"(transcript fork). This is Damien's incident shape — see the test docstring."
)
# And exactly one of the two agents actually rotated its session_id; the
# other should still hold the parent_sid (its compression was skipped).
rotated = sum(
1 for a in (agent_a, agent_b) if a.session_id != parent_sid
)
assert rotated == 1, (
f"Expected exactly one agent to rotate session_id, got {rotated}. "
"Both agents rotating means the lock didn't serialize them."
)
# The lock must be released after the winner finished.
assert db.get_compression_lock_holder(parent_sid) is None, (
"Compression lock leaked: still held after both rotations completed."
)
def test_skipped_compression_returns_messages_unchanged(tmp_path: Path) -> None:
"""The loser of the lock race must return its input messages verbatim.
Callers (preflight compression in ``conversation_loop.py``) detect the
no-op via ``len(returned) == len(input)`` and stop the auto-compress
retry loop. If the skipped path returned the compressed view, that
detection would break and the caller would mutate the conversation
without going through state.db rotation.
"""
db = SessionDB(db_path=tmp_path / "state.db")
parent_sid = "LOSER_TEST"
db.create_session(parent_sid, source="discord")
# Pre-acquire the lock so the agent's compress_context sees it held.
held = db.try_acquire_compression_lock(parent_sid, "external_holder")
assert held is True
agent = _build_agent_with_db(db, parent_sid)
messages = [{"role": "user", "content": "m1"}, {"role": "user", "content": "m2"}]
compressed, _sp = agent._compress_context(messages, "sys", approx_tokens=120_000)
# Skipped: messages returned verbatim, no rotation
assert compressed is messages or compressed == messages
assert agent.session_id == parent_sid
# Compressor was never called (the skip happens before .compress())
agent.context_compressor.compress.assert_not_called()
class _NoLockSubsystemDB:
"""Wraps a real SessionDB but simulates a pre-#34351 version skew.
A long-lived process can hold ``hermes_state.SessionDB`` bound to the
OLD class in memory (no compression-lock methods) while a lazily
re-imported ``conversation_compression.py`` calls the NEW lock code.
``try_acquire_compression_lock`` then raises ``AttributeError`` — which
is NOT a ``sqlite3.Error``, so the method's own fail-open guard never
runs. Before the fix the exception propagated to the outer agent loop,
which printed the error and retried; compression never succeeded, the
token count never dropped, and the loop re-triggered compaction forever.
"""
def __init__(self, real_db: SessionDB) -> None:
self._real = real_db
def try_acquire_compression_lock(self, *_a, **_k): # noqa: D401
raise AttributeError(
"'SessionDB' object has no attribute 'try_acquire_compression_lock'"
)
def get_compression_lock_holder(self, *_a, **_k):
raise AttributeError("'SessionDB' object has no attribute 'get_compression_lock_holder'")
def release_compression_lock(self, *_a, **_k):
raise AttributeError("'SessionDB' object has no attribute 'release_compression_lock'")
def __getattr__(self, name):
# Everything else (create_session, append, rotation helpers) goes to
# the real db so the post-lock compression + rotation path runs.
return getattr(self._real, name)
def test_missing_lock_subsystem_fails_open_not_infinite_loop(tmp_path: Path) -> None:
"""Version skew (no lock methods) must fail OPEN, not raise into the loop.
Reproduces the "API call #47/#48/#49 ... has no attribute
try_acquire_compression_lock" infinite-compaction spin: when the lock
subsystem is absent, ``_compress_context`` must skip locking and proceed
with compression (so the loop makes progress and terminates) instead of
letting the ``AttributeError`` escape to the retry loop.
"""
db = SessionDB(db_path=tmp_path / "state.db")
parent_sid = "SKEW_TEST_SESSION"
db.create_session(parent_sid, source="discord")
agent = _build_agent_with_db(db, parent_sid)
# Swap in the lock-less wrapper AFTER construction (the agent already
# holds a normal db reference; we only break the lock methods).
agent._session_db = _NoLockSubsystemDB(db)
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
# MUST NOT raise AttributeError. Before the fix this raised and the
# outer loop would retry forever.
compressed, _sp = agent._compress_context(messages, "sys", approx_tokens=120_000)
# Compression actually ran (proceeded past the broken lock) and made
# progress, so the auto-compress loop would terminate.
agent.context_compressor.compress.assert_called_once()
assert len(compressed) < len(messages), (
"Compression made no progress despite failing open — loop would still spin."
)
# Session rotated (compression succeeded end-to-end).
assert agent.session_id != parent_sid
def test_review_fork_disables_compression_to_prevent_stale_parent_fork() -> None:
"""The background-review fork must set ``compression_enabled = False``
so it can never compress the parent it shares a session_id with
(issue #38727).
The per-session compression lock only serialises a SAME-WINDOW concurrent
race. It does NOT stop a stale parent from being compressed again in a
LATER turn: if ``review_agent`` had won the race, its new child session is
never adopted by the gateway (the fork is single-lifecycle and dies right
after one ``run_conversation``), so the foreground path would start the
next turn from the stale parent and compress it AGAIN — leaving the same
parent with two sibling children.
The fix makes the review fork never trigger compression at all. Both
compression trigger sites in ``agent/conversation_loop.py`` gate on
``agent.compression_enabled`` BEFORE calling ``_compress_context``:
• preflight (``if agent.compression_enabled and len(messages) > ...``)
• mid-loop (``if agent.compression_enabled and _compressor.should_compress(...)``)
so a fork with the flag cleared never reaches the rotation path.
This test pins the contract at the source: ``_run_review_in_thread``
must set ``review_agent.compression_enabled = False`` on the fork it
builds. It calls the real worker synchronously with
``AIAgent.run_conversation`` patched (so no LLM call happens) and
captures the constructed review agent to assert the flag.
"""
import tempfile
import agent.background_review as br
captured = {}
def _fake_run_conversation(self, *_a, **_k):
captured["compression_enabled"] = self.compression_enabled
captured["session_id"] = self.session_id
return {"final_response": "", "messages": []}
parent_sid = "REVIEW_FORK_FLAG_TEST"
with tempfile.TemporaryDirectory() as td:
db = SessionDB(db_path=Path(td) / "state.db")
db.create_session(parent_sid, source="discord")
parent = _build_agent_with_db(db, parent_sid)
# The worker does a local ``from run_agent import AIAgent``; patching
# the class method covers that import path.
from run_agent import AIAgent
with patch.object(AIAgent, "run_conversation", _fake_run_conversation):
br._run_review_in_thread(
parent,
[{"role": "user", "content": "hi"}],
"review this conversation",
)
assert captured, (
"_run_review_in_thread never reached run_conversation — the spawn path "
"changed; update this test to capture the review AIAgent."
)
assert captured["session_id"] == parent_sid, (
"Review fork should inherit the parent's session_id (shared id is the "
"whole reason compression must be disabled)."
)
assert captured["compression_enabled"] is False, (
"FIX REGRESSION: background-review fork did NOT disable compression. "
"It shares the parent's session_id, so an enabled fork can rotate the "
"parent into an orphan child (issue #38727). The trigger gates in "
"conversation_loop.py only short-circuit when compression_enabled is "
"False — this flag MUST be cleared on the review fork."
)

80
tests/agent/test_compression_logging_session_context.py Normal file
View file

@ -0,0 +1,80 @@
"""Regression: compaction must move the LOGGING session context with the id.
When ``compress_context`` rotates ``agent.session_id`` it updates the
gateway/tools session context (``gateway.session_context.set_current_session_id``,
which moves ``HERMES_SESSION_ID`` env + ContextVar). The ``[session_id]`` tag on
log lines comes from a SEPARATE mechanism — ``hermes_logging._session_context``
(a threading.local read by the global LogRecord factory), set once per turn in
``conversation_loop.py``. Before the fix, the rotation block never updated it, so
log lines emitted after a mid-turn compaction carried the STALE old id while the
message body / session DB / gateway state carried the new one (see #34089). This
asserts the logging context follows the rotation.
"""
from __future__ import annotations
import os
from pathlib import Path
from unittest.mock import MagicMock, patch
import hermes_logging
from hermes_state import SessionDB
def _build_agent_with_db(db: SessionDB, session_id: str):
"""Mirror tests/agent/test_compression_concurrent_fork.py's harness."""
with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
from run_agent import AIAgent
agent = AIAgent(
api_key="test-key",
base_url="https://openrouter.ai/api/v1",
model="test/model",
quiet_mode=True,
session_db=db,
session_id=session_id,
skip_context_files=True,
skip_memory=True,
)
compressor = MagicMock()
compressor.compress.return_value = [
{"role": "user", "content": "[CONTEXT COMPACTION] summary"},
{"role": "user", "content": "tail"},
]
compressor.compression_count = 1
compressor.last_prompt_tokens = 0
compressor.last_completion_tokens = 0
compressor._last_summary_error = None
compressor._last_compress_aborted = False
compressor._last_aux_model_failure_model = None
compressor._last_aux_model_failure_error = None
agent.context_compressor = compressor
return agent
def test_logging_session_context_follows_compression_rotation(tmp_path: Path) -> None:
db = SessionDB(db_path=tmp_path / "state.db")
parent_sid = "PARENT_LOGCTX_SESSION"
db.create_session(parent_sid, source="cli")
agent = _build_agent_with_db(db, parent_sid)
# conversation_loop.py pins the logging tag to the ORIGINAL id at turn start.
hermes_logging.set_session_context(parent_sid)
try:
messages = [{"role": "user", "content": f"m{i}"} for i in range(20)]
agent._compress_context(messages, "sys", approx_tokens=120_000)
# The id actually rotated (sanity — otherwise the assertion is vacuous).
assert agent.session_id != parent_sid
# The logging context must now match the NEW id, not the stale one.
current = getattr(hermes_logging._session_context, "session_id", None)
assert current == agent.session_id, (
"Logging session context did not follow the compaction rotation: "
f"log tag still {current!r}, agent.session_id is {agent.session_id!r} "
"(see #34089)."
)
finally:
hermes_logging.clear_session_context()

1
tests/agent/test_compressor_image_tokens.py
View file

@ -8,7 +8,6 @@ creative workflows that iterate on images across many turns.
from __future__ import annotations
import pytest
from agent.context_compressor import (
_CHARS_PER_TOKEN,

249
tests/agent/test_context_compressor.py
View file

@ -41,6 +41,8 @@ class TestShouldCompress:
class TestUpdateFromResponse:
def test_updates_fields(self, compressor):
compressor.awaiting_real_usage_after_compression = True
compressor.last_compression_rough_tokens = 90_000
compressor.update_from_response({
"prompt_tokens": 5000,
"completion_tokens": 1000,
@ -48,12 +50,39 @@ class TestUpdateFromResponse:
})
assert compressor.last_prompt_tokens == 5000
assert compressor.last_completion_tokens == 1000
assert compressor.last_real_prompt_tokens == 5000
assert compressor.last_rough_tokens_when_real_prompt_fit == 90_000
assert compressor.awaiting_real_usage_after_compression is False
def test_missing_fields_default_zero(self, compressor):
compressor.update_from_response({})
assert compressor.last_prompt_tokens == 0
class TestPreflightDeferral:
def test_defers_when_recent_real_usage_fit_and_rough_growth_is_small(self, compressor):
compressor.threshold_tokens = 85_000
compressor.last_real_prompt_tokens = 50_000
compressor.last_rough_tokens_when_real_prompt_fit = 90_000
assert compressor.should_defer_preflight_to_real_usage(93_000) is True
assert compressor.last_rough_tokens_when_real_prompt_fit == 93_000
def test_does_not_defer_when_rough_growth_is_large(self, compressor):
compressor.threshold_tokens = 85_000
compressor.last_real_prompt_tokens = 50_000
compressor.last_rough_tokens_when_real_prompt_fit = 90_000
assert compressor.should_defer_preflight_to_real_usage(100_000) is False
def test_does_not_defer_without_recent_real_usage(self, compressor):
compressor.threshold_tokens = 85_000
compressor.last_real_prompt_tokens = 0
compressor.last_rough_tokens_when_real_prompt_fit = 90_000
assert compressor.should_defer_preflight_to_real_usage(93_000) is False
class TestCompress:
def _make_messages(self, n):
@ -65,11 +94,12 @@ class TestCompress:
assert result == msgs
def test_truncation_fallback_no_client(self, compressor):
# compressor has client=None and abort_on_summary_failure=False (default),
# so the LEGACY fallback path inserts a static "summary unavailable"
# placeholder and the middle window is dropped.
# Simulate "no summarizer available" explicitly. call_llm can otherwise
# discover the developer's real auxiliary credentials from auth state.
# The failed summary should use the deterministic fallback path.
msgs = [{"role": "system", "content": "System prompt"}] + self._make_messages(10)
result = compressor.compress(msgs)
with patch("agent.context_compressor.call_llm", side_effect=RuntimeError("no provider")):
result = compressor.compress(msgs)
assert len(result) < len(msgs)
# Should keep system message and last N
assert result[0]["role"] == "system"
@ -78,6 +108,64 @@ class TestCompress:
assert compressor._last_compress_aborted is False
assert compressor._last_summary_fallback_used is True
def test_summary_failure_uses_deterministic_fallback_with_recovered_context(self):
"""Regression: failed LLM summaries should not emit a content-free marker.
The fallback should preserve locally recoverable continuity details so a
future turn does not see only "messages were removed" after compaction.
"""
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(
model="test/model",
protect_first_n=1,
protect_last_n=2,
quiet_mode=True,
)
msgs = [
{"role": "system", "content": "System prompt"},
{"role": "user", "content": "Please fix the compression summary failure"},
{
"role": "assistant",
"content": None,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {
"name": "read_file",
"arguments": '{"path":"agent/context_compressor.py","offset":1}',
},
}],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "read agent/context_compressor.py and found static fallback marker",
},
{"role": "assistant", "content": "I found the issue."},
{"role": "user", "content": "latest protected ask"},
{"role": "assistant", "content": "ok"},
]
with (
patch.object(c, "_find_tail_cut_by_tokens", return_value=5),
patch(
"agent.context_compressor.call_llm",
side_effect=RuntimeError("provider down"),
),
):
result = c.compress(msgs)
combined = "\n".join(str(m.get("content", "")) for m in result)
assert "## Active Task" in combined
assert "Please fix the compression summary failure" in combined
assert "read_file" in combined
assert "agent/context_compressor.py" in combined
assert "Summary generation was unavailable" in combined
assert "removed to free context space but could not be summarized" not in combined
assert c._last_summary_fallback_used is True
assert c._last_summary_dropped_count == 3
def test_compression_increments_count(self, compressor):
msgs = self._make_messages(10)
# Default config (abort_on_summary_failure=False) — fallback path
@ -756,6 +844,123 @@ class TestSummaryFailureTrackingForGatewayWarning:
for m in result
)
def test_summary_failure_fallback_preserves_tool_paths_and_redacts_secret_context(self):
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=1, protect_last_n=1)
secret = "ghp_" + ("a" * 36)
msgs = [
{"role": "system", "content": "sys"},
{"role": "user", "content": f"Fix /tmp/project/app.py and never leak {secret}"},
{
"role": "assistant",
"content": "I will inspect it.",
"tool_calls": [
{
"id": "call-1",
"function": {
"name": "read_file",
"arguments": '{"path":"/tmp/project/app.py"}',
},
}
],
},
{"role": "tool", "tool_call_id": "call-1", "content": f"read /tmp/project/app.py with token {secret}"},
{"role": "assistant", "content": "Found the bug in /tmp/project/app.py"},
{"role": "user", "content": "Patch it after this"},
{"role": "assistant", "content": "Ready to patch"},
{"role": "user", "content": "current live request should stay in tail"},
]
with patch("agent.context_compressor.call_llm", side_effect=Exception("timeout")):
result = c.compress(msgs)
fallback = next(m["content"] for m in result if "Summary generation was unavailable" in m.get("content", ""))
assert "Called tool(s): read_file" in fallback
assert "/tmp/project/app.py" in fallback
assert secret not in fallback
assert "ghp_" not in fallback
def test_summary_failure_fallback_supports_object_tool_calls_and_content_path_mentions(self):
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=1, protect_last_n=1)
tool_call = MagicMock()
tool_call.id = "call-object"
tool_call.function.name = "terminal"
tool_call.function.arguments = '{"command":"python /repo/scripts/fix.py", "workdir":"/repo"}'
msgs = [
{"role": "system", "content": "sys"},
{"role": "user", "content": "Review ~/src/pkg/module.py before editing"},
{"role": "assistant", "content": "Running command", "tool_calls": [tool_call]},
{"role": "tool", "tool_call_id": "call-object", "content": "Traceback in /repo/src/pkg/module.py: boom"},
{"role": "assistant", "content": "Need to update C:\\work\\pkg\\module.py too"},
{"role": "user", "content": "Patch ~/src/pkg/module.py after checking those files"},
{"role": "assistant", "content": "Ready to patch"},
{"role": "user", "content": "tail task"},
]
with patch("agent.context_compressor.call_llm", side_effect=Exception("timeout")):
result = c.compress(msgs)
fallback = next(m["content"] for m in result if "Summary generation was unavailable" in m.get("content", ""))
assert "Called tool(s): terminal" in fallback
assert "/repo/scripts/fix.py" in fallback
assert "/repo" in fallback
assert "/repo/src/pkg/module.py" in fallback
assert "C:\\work\\pkg\\module.py" in fallback
assert "Traceback" in fallback
assert "## Last Dropped Turns" in fallback
assert "TOOL: Traceback in /repo/src/pkg/module.py: boom" in fallback
def test_summary_failure_fallback_preserves_last_dropped_turns_without_tail(self):
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=1, protect_last_n=1)
msgs = [
{"role": "system", "content": "sys"},
{"role": "user", "content": "Investigate dropped-window request in /tmp/active.py"},
{"role": "assistant", "content": "I inspected /tmp/active.py and found the failing branch"},
{"role": "tool", "tool_call_id": "call-old", "content": "ValueError: boom in /tmp/active.py"},
{"role": "assistant", "content": "Next step is patching /tmp/active.py"},
{"role": "user", "content": "Confirm regression coverage for /tmp/active.py"},
{"role": "assistant", "content": "Regression note is ready"},
{"role": "user", "content": "protected tail request must not be copied from dropped window"},
]
with patch("agent.context_compressor.call_llm", side_effect=Exception("timeout")):
result = c.compress(msgs)
fallback = next(m["content"] for m in result if "Summary generation was unavailable" in m.get("content", ""))
assert "## Last Dropped Turns" in fallback
assert "ASSISTANT: I inspected /tmp/active.py and found the failing branch" in fallback
assert "TOOL: ValueError: boom in /tmp/active.py" in fallback
assert "protected tail request must not be copied" not in fallback
def test_summary_failure_fallback_is_bounded(self):
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
c = ContextCompressor(model="test", quiet_mode=True, protect_first_n=1, protect_last_n=1)
long_text = "important detail " * 2000
msgs = [
{"role": "system", "content": "sys"},
{"role": "user", "content": "head user"},
{"role": "assistant", "content": "head assistant"},
{"role": "user", "content": long_text},
{"role": "assistant", "content": long_text},
{"role": "user", "content": long_text},
{"role": "assistant", "content": long_text},
{"role": "user", "content": "tail"},
]
with patch("agent.context_compressor.call_llm", side_effect=Exception("timeout")):
result = c.compress(msgs)
fallback = next(m["content"] for m in result if "Summary generation was unavailable" in m.get("content", ""))
assert len(fallback) <= 8300
assert "deterministic fallback" in fallback
assert "important detail" in fallback
def test_compress_clears_fallback_flag_on_subsequent_success(self):
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
@ -1942,3 +2147,39 @@ class TestTruncateToolCallArgsJson:
parsed = _json.loads(shrunk)
assert parsed["path"] == "~/.hermes/skills/shopping/browser-setup-notes.md"
assert parsed["content"].endswith("...[truncated]")
class TestPreflightSentinelGuard:
"""Regression for #36718: the preflight token-display seed in
run_conversation must NOT overwrite the -1 sentinel that
compress_context() sets immediately after compression.
The old guard `_preflight_tokens > (last_prompt_tokens or 0)` evaluated
`(-1 or 0)` -> -1 (truthy), so any positive preflight estimate was > -1
and clobbered the sentinel with a schema-inflated rough count, re-firing
compression on the next turn. The fix treats any negative value as
"no real usage yet" and skips the seed.
"""
def _seed(self, last_prompt_tokens, preflight_tokens):
# Mirror the exact guard in agent/conversation_loop.py run_conversation.
_last = last_prompt_tokens
if _last >= 0 and preflight_tokens > _last:
return preflight_tokens # would overwrite
return last_prompt_tokens # preserved
def test_sentinel_preserved_after_compression(self, compressor):
compressor.last_prompt_tokens = -1
# A large schema-inflated preflight estimate must NOT overwrite -1.
result = self._seed(compressor.last_prompt_tokens, 250_000)
assert result == -1
def test_real_value_still_revises_upward(self, compressor):
compressor.last_prompt_tokens = 10_000
result = self._seed(compressor.last_prompt_tokens, 50_000)
assert result == 50_000
def test_real_value_not_revised_downward(self, compressor):
compressor.last_prompt_tokens = 50_000
result = self._seed(compressor.last_prompt_tokens, 10_000)
assert result == 50_000

145
tests/agent/test_context_compressor_cross_session_guard.py Normal file
View file

@ -0,0 +1,145 @@
"""Tests for cross-session _previous_summary contamination bug (#38788).
ContextCompressor._previous_summary is an instance variable that stores the
previous compaction summary for iterative updates. It is cleared by
on_session_reset() which is called for /new and /reset, but NOT when a cron
session ends naturally. A cron session's compaction sets _previous_summary,
then the cron session ends. A subsequent live messaging session inherits this
stale summary, and _generate_summary() injects it as "PREVIOUS SUMMARY:" into
the summarizer prompt — contaminating the live session's context.
Fix: compress() guards against this by clearing _previous_summary when no
handoff summary is found in the current messages.
"""
import sys
import types
from pathlib import Path
from unittest.mock import patch
# Ensure repo root is importable
sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent))
# Stub out optional heavy dependencies not installed in the test environment
sys.modules.setdefault("fire", types.SimpleNamespace(Fire=lambda *a, **k: None))
sys.modules.setdefault("firecrawl", types.SimpleNamespace(Firecrawl=object))
sys.modules.setdefault("fal_client", types.SimpleNamespace())
from agent.context_compressor import ContextCompressor
def _make_compressor():
"""Build a ContextCompressor with enough state to pass compress() guards."""
c = ContextCompressor.__new__(ContextCompressor)
c.quiet_mode = True
c.model = "test/model"
c.provider = "test"
c.base_url = "http://test"
c.api_key = "test-key"
c.api_mode = ""
c.context_length = 128000
c.threshold_tokens = 64000
c.threshold_percent = 0.50
c.tail_token_budget = 20000
c.protect_last_n = 12
c.summary_model = ""
c.last_prompt_tokens = 100000
c.last_completion_tokens = 0
c._summary_failure_cooldown_until = 0.0
c._max_compaction_summary_tokens = 0
c.summary_budget_tokens = 0
c.abort_on_summary_failure = False
c._last_compress_aborted = False
c._summary_model_fallen_back = False
c.compression_count = 0
c._context_probed = False
c._last_compression_savings_pct = 100.0
c._ineffective_compression_count = 0
c._last_summary_error = None
c._last_summary_dropped_count = 0
c._last_summary_fallback_used = False
c._last_aux_model_failure_error = None
c._last_aux_model_failure_model = None
c.last_real_prompt_tokens = 0
c.last_compression_rough_tokens = 0
c.last_rough_tokens_when_real_prompt_fit = 0
c.awaiting_real_usage_after_compression = False
return c
def _conversation_without_handoff(n_exchanges=12):
"""Build message list with no compaction handoff in it."""
msgs = [{"role": "system", "content": "You are a helpful assistant."}]
for i in range(n_exchanges):
msgs.append({"role": "user", "content": f"Question {i}"})
msgs.append({"role": "assistant", "content": f"Answer {i}"})
return msgs
def _conversation_with_handoff(n_exchanges=12):
"""Build message list WITH a compaction handoff in protected head."""
from agent.context_compressor import SUMMARY_PREFIX
msgs = [{"role": "system", "content": "You are a helpful assistant."}]
msgs.append({"role": "user", "content": SUMMARY_PREFIX + "\nPrevious summary."})
for i in range(n_exchanges):
msgs.append({"role": "user", "content": f"Question {i}"})
msgs.append({"role": "assistant", "content": f"Answer {i}"})
return msgs
def test_stale_previous_summary_cleared_when_no_handoff():
"""Cross-session guard: stale _previous_summary cleared when no handoff."""
c = _make_compressor()
# Simulate state left by a prior cron session's compaction
c._previous_summary = "STALE CRON SUMMARY - this must not leak"
messages = _conversation_without_handoff()
with patch.object(c, "_generate_summary",
return_value="[CONTEXT COMPACTION] Fresh summary."):
result = c.compress(messages)
assert c._previous_summary is None, (
"compress() must clear stale _previous_summary when no handoff "
f"summary exists in current messages. Got: {c._previous_summary!r}"
)
assert result != messages
assert any(
"[CONTEXT COMPACTION]" in (m.get("content", "") or "") for m in result
)
def test_previous_summary_preserved_when_handoff_found():
"""When a handoff IS found, _previous_summary should be preserved for
iterative update within the same session."""
c = _make_compressor()
c._previous_summary = "Summary from earlier compaction in same session"
messages = _conversation_with_handoff()
with patch.object(c, "_generate_summary",
return_value="[CONTEXT COMPACTION] Updated summary."):
c.compress(messages)
# When a handoff IS found, the staleness guard must NOT fire.
# _previous_summary should be updated, not cleared.
assert c._previous_summary is not None, (
"compress() must NOT clear _previous_summary when handoff summary "
"exists in current messages"
)
def test_no_false_positive_when_previous_summary_already_none():
"""When _previous_summary is already None and no handoff found, nothing
should break (the guard is a no-op in this case)."""
c = _make_compressor()
c._previous_summary = None
messages = _conversation_without_handoff()
with patch.object(c, "_generate_summary",
return_value="[CONTEXT COMPACTION] Fresh summary."):
c.compress(messages)
# Should still be None — guard is no-op
assert c._previous_summary is None

18
tests/agent/test_context_compressor_summary_continuity.py
View file

@ -67,3 +67,21 @@ def test_resume_rehydrates_previous_summary_from_handoff_message():
assert "TURNS TO SUMMARIZE:" not in prompt
assert prompt.count(old_summary) == 1
assert f"[USER]: {SUMMARY_PREFIX}" not in prompt
def test_handoff_in_protected_head_populates_previous_summary_before_update():
"""A resumed protected-head handoff should restore iterative-summary state."""
compressor = _compressor()
old_summary = "PROTECTED-HEAD-SUMMARY durable facts from before restart"
seen_turns = []
def fake_generate_summary(turns_to_summarize, focus_topic=None):
seen_turns.extend(turns_to_summarize)
return "new summary from resumed turns"
with patch.object(compressor, "_generate_summary", side_effect=fake_generate_summary):
compressor.compress(_messages_with_handoff(old_summary))
assert compressor._previous_summary == old_summary
assert seen_turns
assert all(old_summary not in str(msg.get("content", "")) for msg in seen_turns)

114
tests/agent/test_context_compressor_temporal_anchoring.py Normal file
View file

@ -0,0 +1,114 @@
"""Tests for temporal anchoring in context-compaction summaries.
The summarizer is handed the current date and instructed to rewrite completed
actions as absolute, dated, past-tense facts (e.g. "email John" ->
"Sent the proposal email to John on 2026-06-07"). This keeps a resumed
conversation from re-issuing work that already happened. Date resolution is
best-effort: a clock failure must omit the rule, never block compaction.
These exercise ``_generate_summary`` directly -- the function that builds the
summarizer prompt. ``test_context_compressor_summary_continuity`` already
proves ``compress()`` routes into ``_generate_summary``.
"""
from datetime import datetime, timezone
from unittest.mock import MagicMock, patch
import hermes_time
from agent.context_compressor import ContextCompressor
def _compressor() -> ContextCompressor:
with patch("agent.context_compressor.get_model_context_length", return_value=100000):
return ContextCompressor(
model="test/model",
threshold_percent=0.85,
protect_first_n=1,
protect_last_n=1,
quiet_mode=True,
)
def _response(content: str):
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
mock_response.choices[0].message.content = content
return mock_response
def _turns():
return [
{"role": "user", "content": "do the first thing"},
{"role": "assistant", "content": "did the first thing"},
{"role": "user", "content": "do the second thing"},
{"role": "assistant", "content": "did the second thing"},
]
def _fixed_now():
return datetime(2026, 6, 7, 12, 0, tzinfo=timezone.utc)
def test_first_compaction_prompt_contains_dated_anchoring_rule():
compressor = _compressor()
assert compressor._previous_summary is None
with patch.object(hermes_time, "now", _fixed_now), patch(
"agent.context_compressor.call_llm", return_value=_response("summary")
) as mock_call:
compressor._generate_summary(_turns())
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
assert "TEMPORAL ANCHORING" in prompt
assert "2026-06-07" in prompt
# The worked example must carry the resolved date, proving interpolation.
assert "Sent the proposal email to John on 2026-06-07" in prompt
# First-compaction path marker still present.
assert "TURNS TO SUMMARIZE:" in prompt
def test_iterative_update_prompt_also_contains_anchoring_rule():
compressor = _compressor()
compressor._previous_summary = "OLD summary body with continuity facts"
with patch.object(hermes_time, "now", _fixed_now), patch(
"agent.context_compressor.call_llm", return_value=_response("updated summary")
) as mock_call:
compressor._generate_summary(_turns())
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
assert "PREVIOUS SUMMARY:" in prompt
assert "TEMPORAL ANCHORING" in prompt
assert "2026-06-07" in prompt
def test_clock_failure_omits_rule_but_compaction_still_runs():
compressor = _compressor()
def _boom():
raise RuntimeError("clock unavailable")
with patch.object(hermes_time, "now", _boom), patch(
"agent.context_compressor.call_llm", return_value=_response("summary")
) as mock_call:
result = compressor._generate_summary(_turns())
# call_llm was still invoked -> compaction was not blocked by the clock error.
assert mock_call.called
assert result is not None
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
assert "TEMPORAL ANCHORING" not in prompt
# Structured template still intact.
assert "## Active Task" in prompt
def test_anchoring_rule_uses_date_from_hermes_time_now():
"""The date is taken from hermes_time.now(), which respects the user's TZ."""
compressor = _compressor()
fixed = datetime(2025, 12, 31, 23, 30, tzinfo=timezone.utc)
with patch.object(hermes_time, "now", lambda: fixed), patch(
"agent.context_compressor.call_llm", return_value=_response("summary")
) as mock_call:
compressor._generate_summary(_turns())
prompt = mock_call.call_args.kwargs["messages"][0]["content"]
assert "2025-12-31" in prompt

2
tests/agent/test_context_engine.py
View file

@ -232,7 +232,7 @@ class TestPluginContextEngineSlot:
assert mgr._context_engine is None
def test_get_plugin_context_engine(self):
from hermes_cli.plugins import PluginManager, PluginContext, PluginManifest, get_plugin_context_engine, _plugin_manager
from hermes_cli.plugins import PluginManager, get_plugin_context_engine
import hermes_cli.plugins as plugins_mod
# Inject a test manager

289
tests/agent/test_context_engine_host_contract.py Normal file
View file

@ -0,0 +1,289 @@
"""Regressions for the context-engine host contract.
These tests pin the five generic host-side guarantees that external context
engine plugins (e.g. hermes-lcm) rely on:
1. ``_transition_context_engine_session`` drives the full lifecycle
(on_session_end → on_session_reset → on_session_start → optional
carry_over_new_session_context) and ``reset_session_state`` delegates
to it when callers pass session metadata.
2. ``on_session_start`` receives ``conversation_id`` derived from
``_gateway_session_key`` at agent init time.
3. ``conversation_loop`` forwards canonical cache buckets
(``cache_read_tokens``, ``cache_write_tokens``, ``input_tokens``,
``output_tokens``, ``reasoning_tokens``) to the engine's
``update_from_response``, on top of the legacy aggregate keys.
4. ``_discover_context_engines`` includes plugin-registered engines (not
just repo-shipped engines under ``plugins/context_engine/``).
5. The repo-shipped ``_EngineCollector`` honors ``ctx.register_command``
from a plugin engine's ``register(ctx)`` entry point and routes it
to the global plugin command registry.
"""
from __future__ import annotations
from unittest.mock import MagicMock
from run_agent import AIAgent
def _bare_agent() -> AIAgent:
agent = object.__new__(AIAgent)
agent.session_id = "test-session"
agent.model = "fake-model"
agent.platform = "telegram"
agent._gateway_session_key = "agent:main:telegram:dm:42"
return agent
def test_transition_runs_full_lifecycle_in_order():
"""End → reset → start → carry_over, in that order, when all inputs apply."""
events: list[str] = []
engine = MagicMock()
engine.context_length = 200_000
engine.on_session_end.side_effect = lambda *a, **kw: events.append("on_session_end")
engine.on_session_reset.side_effect = lambda *a, **kw: events.append("on_session_reset")
engine.on_session_start.side_effect = lambda *a, **kw: events.append("on_session_start")
engine.carry_over_new_session_context.side_effect = lambda *a, **kw: events.append("carry_over")
agent = _bare_agent()
agent.context_compressor = engine
agent._transition_context_engine_session(
old_session_id="old-sid",
new_session_id="new-sid",
previous_messages=[{"role": "user", "content": "hi"}],
carry_over_context=True,
)
assert events == [
"on_session_end",
"on_session_reset",
"on_session_start",
"carry_over",
]
def test_transition_passes_conversation_id_from_gateway_session_key():
"""on_session_start receives ``conversation_id`` from ``_gateway_session_key``."""
engine = MagicMock()
engine.context_length = 200_000
captured: dict = {}
engine.on_session_start.side_effect = lambda sid, **kw: captured.update(kw)
agent = _bare_agent()
agent.context_compressor = engine
agent._transition_context_engine_session(
old_session_id="old-sid",
new_session_id="new-sid",
previous_messages=[{"role": "user", "content": "hi"}],
)
assert captured.get("conversation_id") == "agent:main:telegram:dm:42"
assert captured.get("old_session_id") == "old-sid"
assert captured.get("platform") == "telegram"
def test_transition_skips_optional_hooks_when_engine_lacks_them():
"""Engines that don't implement on_session_end/carry_over still work."""
class MinimalEngine:
def __init__(self):
self.context_length = 100_000
self.reset_called = False
self.start_called_with = None
def on_session_reset(self):
self.reset_called = True
def on_session_start(self, sid, **kw):
self.start_called_with = (sid, kw)
engine = MinimalEngine()
agent = _bare_agent()
agent.context_compressor = engine
# Should not raise even though on_session_end / carry_over are missing.
agent._transition_context_engine_session(
old_session_id="old",
new_session_id="new",
previous_messages=[{"role": "user", "content": "hi"}],
carry_over_context=True,
)
assert engine.reset_called is True
assert engine.start_called_with is not None
new_sid, kw = engine.start_called_with
assert new_sid == "new"
assert kw.get("old_session_id") == "old"
def test_reset_session_state_delegates_to_transition_when_args_provided():
"""``reset_session_state(previous_messages=..., old_session_id=...)`` fires full lifecycle."""
engine = MagicMock()
engine.context_length = 100_000
agent = _bare_agent()
agent.context_compressor = engine
agent.reset_session_state(
previous_messages=[{"role": "user", "content": "hi"}],
old_session_id="old-sid",
)
assert engine.on_session_end.called
assert engine.on_session_reset.called
assert engine.on_session_start.called
# No carry_over_context, so carry_over hook NOT called.
assert not engine.carry_over_new_session_context.called
def test_reset_session_state_default_call_only_resets():
"""Bare ``reset_session_state()`` still only resets the engine (no end/start)."""
engine = MagicMock()
engine.context_length = 100_000
agent = _bare_agent()
agent.context_compressor = engine
agent.reset_session_state()
assert engine.on_session_reset.called
assert not engine.on_session_end.called
assert not engine.on_session_start.called
def test_update_from_response_forwards_canonical_cache_buckets():
"""conversation_loop passes cache_read/write/reasoning tokens to engine."""
# Test the contract directly: a usage_dict built from CanonicalUsage must
# contain the canonical buckets in addition to the legacy keys. We don't
# spin up the full conversation loop; we just verify the dict shape.
from agent.usage_pricing import CanonicalUsage
canonical = CanonicalUsage(
input_tokens=1000,
output_tokens=500,
cache_read_tokens=800,
cache_write_tokens=200,
reasoning_tokens=50,
)
usage_dict = {
"prompt_tokens": canonical.prompt_tokens,
"completion_tokens": canonical.output_tokens,
"total_tokens": canonical.total_tokens,
"input_tokens": canonical.input_tokens,
"output_tokens": canonical.output_tokens,
"cache_read_tokens": canonical.cache_read_tokens,
"cache_write_tokens": canonical.cache_write_tokens,
"reasoning_tokens": canonical.reasoning_tokens,
}
# Legacy keys present
assert usage_dict["prompt_tokens"] == canonical.prompt_tokens
assert usage_dict["completion_tokens"] == 500
assert usage_dict["total_tokens"] == canonical.total_tokens
# Canonical cache + reasoning buckets present
assert usage_dict["cache_read_tokens"] == 800
assert usage_dict["cache_write_tokens"] == 200
assert usage_dict["reasoning_tokens"] == 50
assert usage_dict["input_tokens"] == 1000
assert usage_dict["output_tokens"] == 500
def test_discover_context_engines_includes_plugin_registered_engines(monkeypatch):
"""Plugin-registered context engines appear in the ``hermes plugins`` picker."""
from hermes_cli import plugins_cmd
fake_repo = lambda: [("compressor", "built-in", True)]
class FakePluginEngine:
name = "lcm"
monkeypatch.setattr(
"plugins.context_engine.discover_context_engines",
fake_repo,
)
monkeypatch.setattr(
"hermes_cli.plugins.discover_plugins",
lambda *_a, **_kw: None,
)
monkeypatch.setattr(
"hermes_cli.plugins.get_plugin_context_engine",
lambda: FakePluginEngine(),
)
engines = plugins_cmd._discover_context_engines()
names = [n for n, _desc in engines]
assert "compressor" in names
assert "lcm" in names
def test_discover_context_engines_dedupes_by_name(monkeypatch):
"""Repo-shipped engine wins when name collides with a plugin-registered one."""
from hermes_cli import plugins_cmd
class FakePluginEngine:
name = "compressor" # same name as repo-shipped
monkeypatch.setattr(
"plugins.context_engine.discover_context_engines",
lambda: [("compressor", "built-in compressor", True)],
)
monkeypatch.setattr(
"hermes_cli.plugins.discover_plugins",
lambda *_a, **_kw: None,
)
monkeypatch.setattr(
"hermes_cli.plugins.get_plugin_context_engine",
lambda: FakePluginEngine(),
)
engines = plugins_cmd._discover_context_engines()
# Only one entry — the repo-shipped one. Description is preserved.
assert engines == [("compressor", "built-in compressor")]
def test_engine_collector_forwards_register_command_to_plugin_manager():
"""A plugin context engine can register a slash command via ``ctx.register_command``."""
from plugins.context_engine import _EngineCollector
from hermes_cli.plugins import get_plugin_manager
handler = lambda raw_args: f"echo: {raw_args}"
collector = _EngineCollector(engine_name="my-lcm")
collector.register_command(
"my-lcm-test-cmd",
handler,
description="test command from a context engine",
args_hint="<msg>",
)
manager = get_plugin_manager()
try:
assert "my-lcm-test-cmd" in manager._plugin_commands
entry = manager._plugin_commands["my-lcm-test-cmd"]
assert entry["handler"] is handler
assert entry["args_hint"] == "<msg>"
assert entry["plugin"] == "context-engine:my-lcm"
finally:
# Clean up so we don't leak the registration across tests.
manager._plugin_commands.pop("my-lcm-test-cmd", None)
def test_engine_collector_rejects_builtin_command_conflicts():
"""Context engine cannot shadow built-in slash commands like /help."""
from plugins.context_engine import _EngineCollector
from hermes_cli.plugins import get_plugin_manager
collector = _EngineCollector(engine_name="my-lcm")
collector.register_command("help", lambda *_: "shadow")
manager = get_plugin_manager()
# Must NOT have overwritten / registered against built-in /help.
assert "help" not in manager._plugin_commands or \
manager._plugin_commands["help"].get("plugin") != "context-engine:my-lcm"

27
tests/agent/test_context_references.py
View file

@ -192,21 +192,40 @@ def test_expand_git_diff_staged_and_log(sample_repo: Path):
assert "VALUE = 2" in result.message
def test_binary_and_missing_files_become_warnings(sample_repo: Path):
def test_missing_file_becomes_warning(sample_repo: Path):
from agent.context_references import preprocess_context_references
result = preprocess_context_references(
"Check @file:blob.bin and @file:nope.txt",
"Check @file:nope.txt",
cwd=sample_repo,
context_length=100_000,
)
assert result.expanded
assert len(result.warnings) == 2
assert "binary" in result.message.lower()
assert len(result.warnings) == 1
assert "not found" in result.message.lower()
def test_binary_file_yields_actionable_block_not_a_dead_warning(sample_repo: Path):
from agent.context_references import preprocess_context_references
result = preprocess_context_references(
"Check @file:blob.bin",
cwd=sample_repo,
context_length=100_000,
)
assert result.expanded
# The whole point: a binary attachment must NOT degrade into a discouraging
# warning that makes the model give up — it gets an actionable content block.
assert not result.warnings
assert "blob.bin" in result.message
assert "binary" in result.message.lower()
assert "not supported" not in result.message.lower()
# And it must point the agent at the file so it can act on it with tools.
assert str(sample_repo / "blob.bin") in result.message
def test_soft_budget_warns_and_hard_budget_refuses(sample_repo: Path):
from agent.context_references import preprocess_context_references

914
tests/agent/test_credential_pool.py
View file

@ -379,6 +379,415 @@ def test_mark_exhausted_and_rotate_persists_status(tmp_path, monkeypatch):
assert persisted["last_error_code"] == 402
def test_token_invalidated_marks_credential_dead(tmp_path, monkeypatch):
"""OpenAI Codex token_invalidated must mark the credential DEAD, not exhausted.
Regression for #32849: when an OAuth credential is revoked upstream, the
1-hour exhausted TTL means it re-enters rotation every hour and fails
again with the same 401 — surfacing as "Failed to generate context
summary" on context compression. Terminal OAuth failures should never
auto-recover.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
_write_auth_store(
tmp_path,
{
"version": 1,
"credential_pool": {
"openai-codex": [
{
"id": "cred-dead",
"label": "revoked",
"auth_type": "oauth",
"priority": 0,
"source": "manual:device_code",
"access_token": "revoked-at",
"refresh_token": "revoked-rt",
},
{
"id": "cred-ok",
"label": "healthy",
"auth_type": "oauth",
"priority": 1,
"source": "manual:device_code",
"access_token": "healthy-at",
"refresh_token": "healthy-rt",
},
]
},
},
)
from agent.credential_pool import load_pool, STATUS_DEAD
pool = load_pool("openai-codex")
assert pool.select().id == "cred-dead"
# Simulate the exact OpenAI Codex 401 token_invalidated response shape.
next_entry = pool.mark_exhausted_and_rotate(
status_code=401,
error_context={
"reason": "token_invalidated",
"message": "Your authentication token has been invalidated. Please try signing in again.",
},
)
# Rotation still works — we hand off to the healthy credential.
assert next_entry is not None
assert next_entry.id == "cred-ok"
# The revoked credential is now permanently marked DEAD.
auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text())
persisted = auth_payload["credential_pool"]["openai-codex"][0]
assert persisted["last_status"] == STATUS_DEAD
assert persisted["last_error_code"] == 401
assert persisted["last_error_reason"] == "token_invalidated"
def test_dead_credential_never_re_enters_rotation_after_ttl(tmp_path, monkeypatch):
"""A DEAD credential must stay excluded regardless of how much time passes.
The exhausted TTL clears entries after 5 min (401) / 1 hour (429).
A DEAD credential has no recovery TTL — it stays dead until either
(a) an explicit re-auth write-side sync rewrites the tokens, or
(b) the manual-prune TTL elapses (covered by separate tests below).
This test verifies the core invariant in the recent-entry window.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
# DEAD entry from 2 hours ago — well past the exhausted TTLs (5min/1h)
# but well within the 24h manual-prune window.
two_hours_ago = time.time() - (2 * 3600)
_write_auth_store(
tmp_path,
{
"version": 1,
"credential_pool": {
"openai-codex": [
{
"id": "cred-dead",
"label": "revoked",
"auth_type": "oauth",
"priority": 0,
"source": "manual:device_code",
"access_token": "revoked-at",
"refresh_token": "revoked-rt",
"last_status": "dead",
"last_status_at": two_hours_ago,
"last_error_code": 401,
"last_error_reason": "token_invalidated",
},
{
"id": "cred-ok",
"label": "healthy",
"auth_type": "oauth",
"priority": 1,
"source": "manual:device_code",
"access_token": "healthy-at",
"refresh_token": "healthy-rt",
},
]
},
},
)
from agent.credential_pool import load_pool, STATUS_DEAD
pool = load_pool("openai-codex")
selected = pool.select()
# Should skip the dead entry and pick the healthy one — even though
# the dead entry has priority 0 (would normally be picked first) and
# plenty of time has passed since it was marked dead.
assert selected is not None
assert selected.id == "cred-ok"
# The DEAD entry is still marked dead on disk — not cleared by TTL.
auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text())
dead_entry = next(e for e in auth_payload["credential_pool"]["openai-codex"]
if e["id"] == "cred-dead")
assert dead_entry["last_status"] == STATUS_DEAD
def test_429_rate_limit_still_uses_exhausted_not_dead(tmp_path, monkeypatch):
"""429 rate limits must NOT be treated as terminal.
They should keep the existing 1-hour TTL cooldown semantics so the
credential re-enters rotation once the rate window resets.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
_write_auth_store(
tmp_path,
{
"version": 1,
"credential_pool": {
"openai-codex": [
{
"id": "cred-1",
"label": "primary",
"auth_type": "oauth",
"priority": 0,
"source": "manual:device_code",
"access_token": "at-1",
"refresh_token": "rt-1",
},
{
"id": "cred-2",
"label": "secondary",
"auth_type": "oauth",
"priority": 1,
"source": "manual:device_code",
"access_token": "at-2",
"refresh_token": "rt-2",
},
]
},
},
)
from agent.credential_pool import load_pool, STATUS_EXHAUSTED
pool = load_pool("openai-codex")
assert pool.select().id == "cred-1"
next_entry = pool.mark_exhausted_and_rotate(
status_code=429,
error_context={"reason": "rate_limit_exceeded", "message": "Rate limit exceeded"},
)
assert next_entry is not None
assert next_entry.id == "cred-2"
auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text())
persisted = auth_payload["credential_pool"]["openai-codex"][0]
# 429 stays exhausted (transient) — NOT dead.
assert persisted["last_status"] == STATUS_EXHAUSTED
assert persisted["last_error_code"] == 429
def test_generic_401_without_terminal_reason_still_uses_exhausted(tmp_path, monkeypatch):
"""A 401 with no specific code/reason should keep TTL semantics.
Only specific terminal reasons (token_invalidated, token_revoked, etc.)
transition to DEAD. A generic 401 might be a transient server-side
issue worth retrying after the 5-min TTL.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
_write_auth_store(
tmp_path,
{
"version": 1,
"credential_pool": {
"openai-codex": [
{
"id": "cred-1",
"label": "primary",
"auth_type": "oauth",
"priority": 0,
"source": "manual:device_code",
"access_token": "at-1",
"refresh_token": "rt-1",
},
{
"id": "cred-2",
"label": "secondary",
"auth_type": "oauth",
"priority": 1,
"source": "manual:device_code",
"access_token": "at-2",
"refresh_token": "rt-2",
},
]
},
},
)
from agent.credential_pool import load_pool, STATUS_EXHAUSTED
pool = load_pool("openai-codex")
pool.select()
# 401 with no specific reason — stays exhausted, NOT dead.
pool.mark_exhausted_and_rotate(
status_code=401,
error_context={"message": "Unauthorized"},
)
auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text())
persisted = auth_payload["credential_pool"]["openai-codex"][0]
assert persisted["last_status"] == STATUS_EXHAUSTED
assert persisted["last_error_code"] == 401
def test_dead_manual_entry_pruned_after_24h(tmp_path, monkeypatch):
"""A DEAD manual entry is removed from the pool after the prune TTL.
Manual entries (``manual:*``) are independent credentials with no
singleton to re-seed from, so we can clean them up after a quiet
window without losing recoverability — the user can always re-add
via ``hermes auth add``.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
# DEAD entry from > 24h ago
long_ago = time.time() - (25 * 3600)
_write_auth_store(
tmp_path,
{
"version": 1,
"credential_pool": {
"openai-codex": [
{
"id": "cred-old-dead",
"label": "ancient-dead",
"auth_type": "oauth",
"priority": 0,
"source": "manual:device_code",
"access_token": "stale",
"refresh_token": "stale",
"last_status": "dead",
"last_status_at": long_ago,
"last_error_code": 401,
"last_error_reason": "token_invalidated",
},
{
"id": "cred-ok",
"label": "healthy",
"auth_type": "oauth",
"priority": 1,
"source": "manual:device_code",
"access_token": "healthy-at",
"refresh_token": "healthy-rt",
},
]
},
},
)
from agent.credential_pool import load_pool
pool = load_pool("openai-codex")
# Trigger _available_entries via select; that runs the prune.
selected = pool.select()
assert selected is not None
assert selected.id == "cred-ok"
# On-disk pool should have the dead entry removed.
auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text())
persisted = auth_payload["credential_pool"]["openai-codex"]
assert len(persisted) == 1
assert persisted[0]["id"] == "cred-ok"
def test_dead_manual_entry_kept_within_24h(tmp_path, monkeypatch):
"""A DEAD manual entry stays in the pool until the prune TTL elapses.
Recent DEAD entries are kept so the audit trail (last_error_reason,
timestamps) remains visible while the user investigates. They simply
don't participate in rotation (covered by the DEAD-skip test above).
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
# DEAD entry from only an hour ago — well within the 24h window
recent = time.time() - 3600
_write_auth_store(
tmp_path,
{
"version": 1,
"credential_pool": {
"openai-codex": [
{
"id": "cred-recent-dead",
"label": "recent-dead",
"auth_type": "oauth",
"priority": 0,
"source": "manual:device_code",
"access_token": "stale",
"refresh_token": "stale",
"last_status": "dead",
"last_status_at": recent,
"last_error_code": 401,
"last_error_reason": "token_invalidated",
},
{
"id": "cred-ok",
"label": "healthy",
"auth_type": "oauth",
"priority": 1,
"source": "manual:device_code",
"access_token": "healthy-at",
"refresh_token": "healthy-rt",
},
]
},
},
)
from agent.credential_pool import load_pool, STATUS_DEAD
pool = load_pool("openai-codex")
selected = pool.select()
assert selected is not None
assert selected.id == "cred-ok"
# On-disk pool should still have BOTH entries — recent dead is preserved.
auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text())
persisted = auth_payload["credential_pool"]["openai-codex"]
assert len(persisted) == 2
dead_entry = next(e for e in persisted if e["id"] == "cred-recent-dead")
assert dead_entry["last_status"] == STATUS_DEAD
def test_dead_singleton_seeded_entry_not_pruned(tmp_path, monkeypatch):
"""A DEAD ``device_code`` entry must NOT be pruned even after 24h.
Singleton-seeded entries get re-created by ``_seed_from_singletons`` on
every ``load_pool()``, so pruning them is pointless — they reappear
immediately with the same stale singleton tokens. Keep them visible
with the DEAD marker so the user knows what's broken.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
long_ago = time.time() - (48 * 3600)
_write_auth_store(
tmp_path,
{
"version": 1,
"providers": {
"openai-codex": {
"tokens": {"access_token": "revoked-at", "refresh_token": "revoked-rt"},
"last_refresh": "2026-01-01T00:00:00Z",
"auth_mode": "chatgpt",
},
},
"credential_pool": {
"openai-codex": [
{
"id": "cred-seeded-dead",
"label": "seeded-dead",
"auth_type": "oauth",
"priority": 0,
"source": "device_code", # singleton-seeded, NOT manual
"access_token": "revoked-at",
"refresh_token": "revoked-rt",
"last_status": "dead",
"last_status_at": long_ago,
"last_error_code": 401,
"last_error_reason": "token_invalidated",
},
]
},
},
)
from agent.credential_pool import load_pool, STATUS_DEAD
pool = load_pool("openai-codex")
# No healthy entry available; select returns None (pool empty for rotation).
assert pool.select() is None
# On-disk: the singleton-seeded DEAD entry is preserved.
auth_payload = json.loads((tmp_path / "hermes" / "auth.json").read_text())
persisted = auth_payload["credential_pool"]["openai-codex"]
assert len(persisted) == 1
assert persisted[0]["id"] == "cred-seeded-dead"
assert persisted[0]["last_status"] == STATUS_DEAD
def test_load_pool_seeds_env_api_key(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-seeded")
@ -395,6 +804,324 @@ def test_load_pool_seeds_env_api_key(tmp_path, monkeypatch):
def test_load_pool_does_not_persist_env_seeded_secret_value(tmp_path, monkeypatch):
"""Runtime env keys may be used in memory but must not land in auth.json."""
sentinel = "S3NTINEL_DO_NOT_PERSIST_OPENROUTER"
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
monkeypatch.setenv("OPENROUTER_API_KEY", sentinel)
_write_auth_store(tmp_path, {"version": 1, "providers": {}})
from agent.credential_pool import load_pool
pool = load_pool("openrouter")
entry = pool.select()
assert entry is not None
assert entry.source == "env:OPENROUTER_API_KEY"
assert entry.access_token == sentinel
auth_text = (tmp_path / "hermes" / "auth.json").read_text()
assert sentinel not in auth_text
persisted = json.loads(auth_text)["credential_pool"]["openrouter"][0]
assert persisted["source"] == "env:OPENROUTER_API_KEY"
assert persisted["label"] == "OPENROUTER_API_KEY"
assert persisted["auth_type"] == "api_key"
assert persisted["priority"] == 0
assert "access_token" not in persisted
assert persisted["secret_fingerprint"].startswith("sha256:")
def test_load_pool_persists_bitwarden_origin_metadata_without_secret(tmp_path, monkeypatch):
"""Bitwarden-injected env vars retain source metadata but not raw values."""
sentinel = "S3NTINEL_DO_NOT_PERSIST_BITWARDEN"
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
monkeypatch.setenv("OPENROUTER_API_KEY", sentinel)
monkeypatch.setattr(
"hermes_cli.env_loader.get_secret_source",
lambda env_var: "bitwarden" if env_var == "OPENROUTER_API_KEY" else None,
)
_write_auth_store(tmp_path, {"version": 1, "providers": {}})
from agent.credential_pool import load_pool
pool = load_pool("openrouter")
entry = pool.select()
assert entry is not None
assert entry.access_token == sentinel
assert entry.source == "env:OPENROUTER_API_KEY"
auth_text = (tmp_path / "hermes" / "auth.json").read_text()
assert sentinel not in auth_text
persisted = json.loads(auth_text)["credential_pool"]["openrouter"][0]
assert persisted["source"] == "env:OPENROUTER_API_KEY"
assert persisted["secret_source"] == "bitwarden"
assert "access_token" not in persisted
def test_load_pool_sanitizes_legacy_raw_borrowed_entry_when_value_unchanged(tmp_path, monkeypatch):
"""Existing raw env-seeded pool entries are rewritten even if the env value matches."""
sentinel = "S3NTINEL_DO_NOT_PERSIST_LEGACY_RAW"
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
monkeypatch.setenv("OPENROUTER_API_KEY", sentinel)
_write_auth_store(
tmp_path,
{
"version": 1,
"credential_pool": {
"openrouter": [
{
"id": "legacy-env",
"label": "OPENROUTER_API_KEY",
"auth_type": "api_key",
"priority": 0,
"source": "env:OPENROUTER_API_KEY",
"access_token": sentinel,
"base_url": "https://openrouter.ai/api/v1",
}
]
},
},
)
from agent.credential_pool import load_pool
pool = load_pool("openrouter")
entry = pool.select()
assert entry is not None
assert entry.access_token == sentinel
auth_text = (tmp_path / "hermes" / "auth.json").read_text()
assert sentinel not in auth_text
persisted = json.loads(auth_text)["credential_pool"]["openrouter"][0]
assert persisted["id"] == "legacy-env"
assert "access_token" not in persisted
assert persisted["secret_fingerprint"].startswith("sha256:")
def test_pooled_credential_to_dict_strips_borrowed_secret_fields():
from agent.credential_pool import PooledCredential
sentinel = "S3NTINEL_DO_NOT_PERSIST_TO_DICT"
credential = PooledCredential(
provider="openrouter",
id="borrowed-1",
label="vault-ref",
auth_type="api_key",
priority=3,
source="vault:openrouter/api-key",
access_token=sentinel,
refresh_token=f"refresh-{sentinel}",
agent_key=f"agent-{sentinel}",
request_count=7,
last_status="ok",
extra={
"api_key": f"extra-{sentinel}",
"client_secret": f"client-{sentinel}",
"secret_key": f"secret-key-{sentinel}",
"authToken": f"auth-token-{sentinel}",
"refreshToken": f"camel-refresh-{sentinel}",
"authorization": f"Bearer {sentinel}",
"tokens": {"access_token": f"nested-{sentinel}"},
"token_type": "Bearer",
"scope": "inference",
},
)
payload = credential.to_dict()
serialized = json.dumps(payload)
assert sentinel not in serialized
assert "access_token" not in payload
assert "refresh_token" not in payload
assert "agent_key" not in payload
assert "api_key" not in payload
assert "client_secret" not in payload
assert "secret_key" not in payload
assert "authToken" not in payload
assert "refreshToken" not in payload
assert "authorization" not in payload
assert "tokens" not in payload
assert payload["source"] == "vault:openrouter/api-key"
assert payload["label"] == "vault-ref"
assert payload["request_count"] == 7
assert payload["token_type"] == "Bearer"
assert payload["scope"] == "inference"
assert payload["secret_fingerprint"].startswith("sha256:")
@pytest.mark.parametrize("source", [
"age://openrouter/api-key",
"systemd",
"keyring",
"1password",
"pass",
"sops",
"future_secret_store:openrouter",
])
def test_borrowed_source_variants_strip_secret_fields(source):
from agent.credential_pool import PooledCredential
sentinel = f"S3NTINEL_DO_NOT_PERSIST_{source.replace(':', '_').replace('/', '_')}"
credential = PooledCredential(
provider="openrouter",
id="borrowed-variant",
label="borrowed",
auth_type="api_key",
priority=0,
source=source,
access_token=sentinel,
refresh_token=f"refresh-{sentinel}",
)
payload = credential.to_dict()
serialized = json.dumps(payload)
assert sentinel not in serialized
assert "access_token" not in payload
assert "refresh_token" not in payload
assert payload["source"] == source
assert payload["secret_fingerprint"].startswith("sha256:")
def test_load_pool_prunes_stale_borrowed_custom_config_entry(tmp_path, monkeypatch):
sentinel = "S3NTINEL_DO_NOT_PERSIST_STALE_CUSTOM"
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
_write_auth_store(
tmp_path,
{
"version": 1,
"credential_pool": {
"custom:foo": [
{
"id": "stale-custom",
"label": "Foo",
"auth_type": "api_key",
"priority": 0,
"source": "config:Foo",
"access_token": sentinel,
"base_url": "https://foo.example/v1",
}
]
},
},
)
from agent.credential_pool import load_pool
pool = load_pool("custom:foo")
assert pool.entries() == []
auth_text = (tmp_path / "hermes" / "auth.json").read_text()
assert sentinel not in auth_text
assert json.loads(auth_text)["credential_pool"]["custom:foo"] == []
def test_write_credential_pool_sanitizes_borrowed_payload_at_disk_boundary(tmp_path, monkeypatch):
"""Direct dictionary callers cannot bypass the borrowed-secret guard."""
sentinel = "S3NTINEL_DO_NOT_PERSIST_DIRECT_WRITE"
manual_secret = "MANUAL_SECRET_STAYS_PERSISTABLE"
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
from hermes_cli.auth import write_credential_pool
write_credential_pool("openrouter", [
{
"id": "borrowed-1",
"label": "systemd-ref",
"auth_type": "api_key",
"priority": 0,
"source": "systemd://hermes/openrouter",
"access_token": sentinel,
"refresh_token": f"refresh-{sentinel}",
"agent_key": f"agent-{sentinel}",
"api_key": f"extra-{sentinel}",
},
{
"id": "manual-1",
"label": "manual",
"auth_type": "api_key",
"priority": 1,
"source": "manual",
"access_token": manual_secret,
},
])
auth_text = (tmp_path / "hermes" / "auth.json").read_text()
assert sentinel not in auth_text
assert manual_secret in auth_text
entries = json.loads(auth_text)["credential_pool"]["openrouter"]
borrowed, manual = entries
assert borrowed["source"] == "systemd://hermes/openrouter"
assert "access_token" not in borrowed
assert "refresh_token" not in borrowed
assert "agent_key" not in borrowed
assert "api_key" not in borrowed
assert borrowed["secret_fingerprint"].startswith("sha256:")
assert manual["access_token"] == manual_secret
def test_write_credential_pool_treats_unowned_oauth_source_as_borrowed(tmp_path, monkeypatch):
sentinel = "S3NTINEL_DO_NOT_PERSIST_UNOWNED_OAUTH"
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
from hermes_cli.auth import write_credential_pool
write_credential_pool("openrouter", [
{
"id": "unowned-oauth",
"label": "unowned-oauth",
"auth_type": "oauth",
"priority": 0,
"source": "oauth",
"access_token": sentinel,
"refresh_token": f"refresh-{sentinel}",
}
])
auth_text = (tmp_path / "hermes" / "auth.json").read_text()
assert sentinel not in auth_text
persisted = json.loads(auth_text)["credential_pool"]["openrouter"][0]
assert persisted["source"] == "oauth"
assert "access_token" not in persisted
assert "refresh_token" not in persisted
assert persisted["secret_fingerprint"].startswith("sha256:")
def test_write_credential_pool_preserves_known_provider_owned_oauth_state(tmp_path, monkeypatch):
sentinel = "PROVIDER_OWNED_DEVICE_CODE_STAYS_PERSISTABLE"
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
from hermes_cli.auth import write_credential_pool
write_credential_pool("nous", [
{
"id": "nous-device",
"label": "device-code",
"auth_type": "oauth",
"priority": 0,
"source": "device_code",
"access_token": sentinel,
"refresh_token": f"refresh-{sentinel}",
"agent_key": f"agent-{sentinel}",
}
])
persisted = json.loads((tmp_path / "hermes" / "auth.json").read_text())["credential_pool"]["nous"][0]
assert persisted["access_token"] == sentinel
assert persisted["refresh_token"] == f"refresh-{sentinel}"
assert persisted["agent_key"] == f"agent-{sentinel}"
def test_load_pool_prefers_dotenv_over_stale_os_environ(tmp_path, monkeypatch):
"""Regression for #18254: stale OPENROUTER_API_KEY in os.environ (inherited
from a parent shell) must NOT shadow the fresh key in ~/.hermes/.env when
@ -498,7 +1225,7 @@ def test_load_pool_migrates_nous_provider_state(tmp_path, monkeypatch):
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": "access-token",
"refresh_token": "refresh-token",
"expires_at": "2026-03-24T12:00:00+00:00",
@ -525,7 +1252,7 @@ def test_load_pool_mirrors_nous_invoke_jwt_agent_key_runtime_api_key(tmp_path, m
expires_at = datetime.fromtimestamp(time.time() + 3600, tz=timezone.utc).isoformat()
token = _jwt_with_claims({
"sub": "test-user",
"scope": ["inference:invoke", "inference:mint_agent_key"],
"scope": ["inference:invoke"],
"exp": int(time.time() + 3600),
})
_write_auth_store(
@ -539,7 +1266,7 @@ def test_load_pool_mirrors_nous_invoke_jwt_agent_key_runtime_api_key(tmp_path, m
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:invoke inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": token,
"refresh_token": "refresh-token",
"expires_at": expires_at,
@ -566,6 +1293,29 @@ def test_load_pool_mirrors_nous_invoke_jwt_agent_key_runtime_api_key(tmp_path, m
assert pool_entry["agent_key_expires_at"] == expires_at
def test_nous_runtime_api_key_rejects_opaque_agent_key():
from agent.credential_pool import PooledCredential
entry = PooledCredential(
provider="nous",
id="nous-opaque",
label="opaque",
auth_type="oauth",
priority=0,
source="device_code",
access_token="opaque-access-token",
refresh_token="refresh-token",
agent_key="opaque-agent-key",
agent_key_expires_at=datetime.fromtimestamp(
time.time() + 3600,
tz=timezone.utc,
).isoformat(),
extra={"scope": "inference:invoke"},
)
assert entry.runtime_api_key == ""
def test_nous_pool_terminal_refresh_removes_device_code_entry(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
monkeypatch.setenv("HERMES_SHARED_AUTH_DIR", str(tmp_path / "shared"))
@ -580,7 +1330,7 @@ def test_nous_pool_terminal_refresh_removes_device_code_entry(tmp_path, monkeypa
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": "access-token",
"refresh_token": "refresh-token",
"expires_at": "2026-03-24T12:00:00+00:00",
@ -752,7 +1502,7 @@ def test_load_pool_migrates_nous_provider_state_preserves_tls(tmp_path, monkeypa
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": "access-token",
"refresh_token": "refresh-token",
"expires_at": "2026-03-24T12:00:00+00:00",
@ -864,6 +1614,150 @@ def test_load_pool_prefers_anthropic_env_token_over_file_backed_oauth(tmp_path,
assert entry.access_token == "env-override-token"
def test_load_pool_api_key_path_skips_oauth_autodiscovery(tmp_path, monkeypatch):
"""API-key auth path: autodiscovered OAuth creds must NOT be seeded.
When the user picks "Anthropic API key" at `hermes setup`,
`save_anthropic_api_key()` writes ANTHROPIC_API_KEY and zeros
ANTHROPIC_TOKEN. That env-var pattern is the explicit signal that the
user opted into the API-key path and explicitly OUT of the OAuth
masquerade (Claude Code identity injection + `mcp_` tool-name rewrite
+ claude-cli user-agent). Autodiscovered Claude Code / Hermes PKCE
tokens from other tools' credential files must NOT be silently mixed
into the anthropic pool — otherwise rotation on a 401/429 could flip
the session onto OAuth credentials mid-conversation.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-api03-explicit-user-key")
monkeypatch.delenv("ANTHROPIC_TOKEN", raising=False)
monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False)
_write_auth_store(tmp_path, {"version": 1, "providers": {}})
monkeypatch.setattr("hermes_cli.auth.is_provider_explicitly_configured", lambda pid: True)
pkce_called = {"n": 0}
cc_called = {"n": 0}
def _fake_pkce():
pkce_called["n"] += 1
return {
"accessToken": "sk-ant-oat01-pkce-token",
"refreshToken": "pkce-refresh",
"expiresAt": int(time.time() * 1000) + 3_600_000,
}
def _fake_cc():
cc_called["n"] += 1
return {
"accessToken": "sk-ant-oat01-claude-code-token",
"refreshToken": "cc-refresh",
"expiresAt": int(time.time() * 1000) + 3_600_000,
}
monkeypatch.setattr("agent.anthropic_adapter.read_hermes_oauth_credentials", _fake_pkce)
monkeypatch.setattr("agent.anthropic_adapter.read_claude_code_credentials", _fake_cc)
from agent.credential_pool import load_pool
pool = load_pool("anthropic")
sources = {entry.source for entry in pool.entries()}
# Only the explicit API-key entry should be in the pool.
assert sources == {"env:ANTHROPIC_API_KEY"}, f"got {sources}"
# And we should not have even called the autodiscovery readers.
assert pkce_called["n"] == 0
assert cc_called["n"] == 0
def test_load_pool_api_key_path_prunes_stale_oauth_entries(tmp_path, monkeypatch):
"""Switching OAuth -> API key must prune stale OAuth entries from auth.json.
Without this, a user who logs into OAuth (seeding `claude_code` or
`hermes_pkce` into auth.json) and later switches to the API key at
`hermes setup` would still have those OAuth entries dormant on disk.
Pool rotation on a transient 401 could revive them and flip the
session onto the OAuth masquerade.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-api03-explicit-user-key")
monkeypatch.delenv("ANTHROPIC_TOKEN", raising=False)
monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False)
# Plant a stale claude_code entry in the on-disk pool (as if a previous
# OAuth session seeded it).
_write_auth_store(
tmp_path,
{
"version": 1,
"providers": {},
"credential_pool": {
"anthropic": [
{
"id": "stale1",
"source": "claude_code",
"auth_type": "oauth",
"access_token": "sk-ant-oat01-stale-claude-code",
"refresh_token": "stale-refresh",
"expires_at_ms": int(time.time() * 1000) + 3_600_000,
"priority": 0,
"label": "stale-claude-code",
"request_count": 0,
},
],
},
},
)
monkeypatch.setattr("hermes_cli.auth.is_provider_explicitly_configured", lambda pid: True)
monkeypatch.setattr("agent.anthropic_adapter.read_hermes_oauth_credentials", lambda: None)
monkeypatch.setattr("agent.anthropic_adapter.read_claude_code_credentials", lambda: None)
from agent.credential_pool import load_pool
pool = load_pool("anthropic")
sources = {entry.source for entry in pool.entries()}
# Stale claude_code entry must be gone, API key must be present.
assert "claude_code" not in sources
assert "env:ANTHROPIC_API_KEY" in sources
def test_load_pool_oauth_path_still_autodiscovers(tmp_path, monkeypatch):
"""OAuth path: ANTHROPIC_TOKEN set, autodiscovery still fires.
Regression guard: the API-key gate must not affect users who chose the
OAuth path at `hermes setup`. When ANTHROPIC_TOKEN is set (and
ANTHROPIC_API_KEY is empty), autodiscovered Claude Code creds should
still be seeded into the pool as before.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
monkeypatch.setenv("ANTHROPIC_TOKEN", "sk-ant-oat01-explicit-oauth-token")
monkeypatch.delenv("CLAUDE_CODE_OAUTH_TOKEN", raising=False)
_write_auth_store(tmp_path, {"version": 1, "providers": {}})
monkeypatch.setattr("hermes_cli.auth.is_provider_explicitly_configured", lambda pid: True)
monkeypatch.setattr(
"agent.anthropic_adapter.read_hermes_oauth_credentials",
lambda: None,
)
monkeypatch.setattr(
"agent.anthropic_adapter.read_claude_code_credentials",
lambda: {
"accessToken": "sk-ant-oat01-autodiscovered-cc",
"refreshToken": "cc-refresh",
"expiresAt": int(time.time() * 1000) + 3_600_000,
},
)
from agent.credential_pool import load_pool
pool = load_pool("anthropic")
sources = {entry.source for entry in pool.entries()}
# Both env OAuth token and autodiscovered Claude Code creds should be there.
assert "env:ANTHROPIC_TOKEN" in sources
assert "claude_code" in sources
def test_least_used_strategy_selects_lowest_count(tmp_path, monkeypatch):
"""least_used strategy should select the credential with the lowest request_count."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "hermes"))
@ -1534,7 +2428,7 @@ def test_sync_nous_entry_from_auth_store_adopts_newer_tokens(tmp_path, monkeypat
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": "access-OLD",
"refresh_token": "refresh-OLD",
"expires_at": "2026-03-24T12:00:00+00:00",
@ -1564,7 +2458,7 @@ def test_sync_nous_entry_from_auth_store_adopts_newer_tokens(tmp_path, monkeypat
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": "access-NEW",
"refresh_token": "refresh-NEW",
"expires_at": "2026-03-24T12:30:00+00:00",
@ -1596,7 +2490,7 @@ def test_sync_nous_entry_noop_when_tokens_match(tmp_path, monkeypatch):
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": "access-token",
"refresh_token": "refresh-token",
"expires_at": "2026-03-24T12:00:00+00:00",
@ -1633,7 +2527,7 @@ def test_nous_exhausted_entry_recovers_via_auth_store_sync(tmp_path, monkeypatch
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": "access-OLD",
"refresh_token": "refresh-OLD",
"expires_at": "2026-03-24T12:00:00+00:00",
@ -1670,7 +2564,7 @@ def test_nous_exhausted_entry_recovers_via_auth_store_sync(tmp_path, monkeypatch
"inference_base_url": "https://inference.example.com/v1",
"client_id": "hermes-cli",
"token_type": "Bearer",
"scope": "inference:mint_agent_key",
"scope": "inference:invoke",
"access_token": "access-FRESH",
"refresh_token": "refresh-FRESH",
"expires_at": "2026-03-24T12:30:00+00:00",

207
tests/agent/test_credits_cold_start.py Normal file
View file

@ -0,0 +1,207 @@
"""Tests for cold-start credits hydration at session open.
The L3 cold-start seed primes agent._credits_state from /api/oauth/account (or a
HERMES_DEV_CREDITS_FIXTURE) so depletion AND the 90% grant warning fire immediately
at session open, not only after the first inference header. These tests assert the
notice policy fires correctly for a seed-shaped CreditsState with the warn90 latch
primed the way conversation_loop does it.
"""
import time
from agent.credits_tracker import CreditsState, evaluate_credits_notices
def _cold_start_notices(state: CreditsState):
"""Mirror the conversation_loop seed: prime seen_below_90 when used_fraction is
computable (the snapshot IS the first observation), then evaluate once."""
latch = {"active": set(), "seen_below_90": False}
if state.used_fraction is not None:
latch["seen_below_90"] = True
show, clear = evaluate_credits_notices(state, latch)
return [n.key for n in show]
def _state(**kw) -> CreditsState:
kw.setdefault("from_header", False)
kw.setdefault("captured_at", time.time())
return CreditsState(**kw)
def test_cold_start_healthy_no_notice():
s = _state(
remaining_micros=30_340_000, subscription_micros=18_000_000,
subscription_limit_micros=20_000_000, subscription_limit_usd="20.00",
denominator_kind="subscription_cap", paid_access=True,
)
assert abs(s.used_fraction - 0.1) < 1e-9
assert _cold_start_notices(s) == []
def test_cold_start_opens_already_at_90pct_warns():
"""A session that OPENS already ≥90% must warn immediately — the seed primes
seen_below_90 so warn90 fires without a prior live crossing."""
s = _state(
remaining_micros=2_000_000, subscription_micros=2_000_000,
subscription_limit_micros=20_000_000, subscription_limit_usd="20.00",
denominator_kind="subscription_cap", paid_access=True,
)
assert s.used_fraction == 0.9
assert "credits.usage" in _cold_start_notices(s)
def test_cold_start_grant_exhausted_warns_and_grant_spent():
s = _state(
remaining_micros=12_340_000, subscription_micros=0,
subscription_limit_micros=20_000_000, subscription_limit_usd="20.00",
purchased_micros=12_340_000, denominator_kind="subscription_cap", paid_access=True,
)
assert s.used_fraction == 1.0
keys = _cold_start_notices(s)
assert "credits.usage" in keys
assert "credits.grant_spent" in keys
def test_cold_start_depleted_warns():
s = _state(
remaining_micros=0, subscription_micros=0, purchased_micros=0,
paid_access=False, disabled_reason="out_of_credits",
)
assert s.used_fraction is None # no cap → no %, depletion keys off paid_access
assert _cold_start_notices(s) == ["credits.depleted"]
def test_cold_start_debt_warns_and_depleted():
"""Negative subscription balance (the only signed field) → 100% used + depleted."""
s = _state(
remaining_micros=0, subscription_micros=-5_000_000,
subscription_limit_micros=20_000_000, subscription_limit_usd="20.00",
denominator_kind="subscription_cap", paid_access=False,
disabled_reason="out_of_credits",
)
assert s.used_fraction == 1.0
keys = _cold_start_notices(s)
assert "credits.usage" in keys
assert "credits.depleted" in keys
def test_cold_start_no_cap_degrades_to_depletion_only():
"""Without monthly_credits (older portals) the seed sets no limit → used_fraction
None → only depletion can fire, never warn90."""
healthy_no_cap = _state(
remaining_micros=30_000_000, subscription_micros=18_000_000,
subscription_limit_micros=None, denominator_kind="none", paid_access=True,
)
assert healthy_no_cap.used_fraction is None
assert _cold_start_notices(healthy_no_cap) == []
def test_dev_fixtures_drive_cold_start():
"""Every HERMES_DEV_CREDITS_FIXTURE state produces a valid seed CreditsState."""
import os
from agent.credits_tracker import dev_fixture_credits_state
expected = {
"healthy": [],
"sub_90pct": ["credits.usage"],
"depleted": ["credits.depleted"],
}
for name, want in expected.items():
os.environ["HERMES_DEV_CREDITS"] = "1" # fixtures gate on the dev flag
os.environ["HERMES_DEV_CREDITS_FIXTURE"] = name
try:
fx = dev_fixture_credits_state()
assert fx is not None, name
assert _cold_start_notices(fx) == want, (name, _cold_start_notices(fx))
finally:
os.environ.pop("HERMES_DEV_CREDITS_FIXTURE", None)
os.environ.pop("HERMES_DEV_CREDITS", None)
# ── seed_credits_at_session_start: the shared session-open hydrator ───────────
class _FakeAgent:
"""Minimal agent surface for the seed helper: state slots + an emit that runs
the real policy against the latch (mirroring run_agent._emit_credits_notices,
including the free-model suppression flag)."""
def __init__(self, provider="nous", model=""):
from agent.credits_tracker import evaluate_credits_notices, is_free_tier_model
self.provider = provider
self.model = model
self._credits_state = None
self._credits_session_start_micros = None
self._credits_latch = {"active": set(), "seen_below_90": False, "usage_band": None}
self.emitted: list = []
self._eval = evaluate_credits_notices
self._is_free = is_free_tier_model
def _emit_credits_notices(self):
if self._credits_state is None:
return
show, clear = self._eval(
self._credits_state,
self._credits_latch,
model_is_free=self._is_free(self.model),
)
self.emitted.append(([n.key for n in show], clear))
def _seed(agent, fixture):
import os
from agent.credits_tracker import seed_credits_at_session_start
os.environ["HERMES_DEV_CREDITS"] = "1" # fixtures gate on the dev flag
os.environ["HERMES_DEV_CREDITS_FIXTURE"] = fixture
try:
return seed_credits_at_session_start(agent)
finally:
os.environ.pop("HERMES_DEV_CREDITS_FIXTURE", None)
os.environ.pop("HERMES_DEV_CREDITS", None)
def test_seed_fires_usage_band_at_session_open():
a = _FakeAgent()
assert _seed(a, "sub_90pct") is True
assert a._credits_state is not None
assert a.emitted == [(["credits.usage"], [])]
def test_seed_fires_depleted_at_session_open():
a = _FakeAgent()
assert _seed(a, "depleted") is True
assert a.emitted == [(["credits.depleted"], [])]
def test_seed_depleted_suppressed_on_free_model():
"""A session that opens depleted but on a Nous ``:free`` model must NOT show
the depleted banner — inference works fine on the free tier."""
a = _FakeAgent(model="nvidia/nemotron-3-ultra:free")
assert _seed(a, "depleted") is True
assert a.emitted == [([], [])]
def test_seed_healthy_no_notice():
a = _FakeAgent()
assert _seed(a, "healthy") is True
assert a.emitted == [([], [])]
def test_seed_is_idempotent():
a = _FakeAgent()
_seed(a, "sub_90pct")
a.emitted = []
# second call must no-op (state already populated)
assert _seed(a, "sub_90pct") is False
assert a.emitted == []
def test_seed_skips_non_nous():
from agent.credits_tracker import seed_credits_at_session_start
a = _FakeAgent(provider="openrouter")
assert seed_credits_at_session_start(a) is False
assert a._credits_state is None

67
tests/agent/test_credits_fixture_snapshot.py Normal file
View file

@ -0,0 +1,67 @@
"""Tests for _snapshot_from_credits_state — the dev-fixture /usage renderer.
``build_nous_credits_snapshot`` maps a live portal account; ``_snapshot_from_credits_state``
maps a header-shaped CreditsState (e.g. a HERMES_DEV_CREDITS_FIXTURE) into the SAME
/usage snapshot shape, so the gauge + magnitudes are exercisable offline. These lock
the gauge math, the verbatim *_usd magnitudes (never parseFloat'd), the depletion line,
and the dev-fixture marker.
"""
from __future__ import annotations
from agent.account_usage import _snapshot_from_credits_state
from agent.credits_tracker import CreditsState
def _state(**kw) -> CreditsState:
kw.setdefault("from_header", True)
return CreditsState(**kw)
def test_renders_gauge_magnitudes_and_fixture_marker():
# used_fraction = (20 - 10) / 20 = 0.5 → a 50%-used gauge window
snap = _snapshot_from_credits_state(_state(
remaining_micros=30_340_000, remaining_usd="30.34",
subscription_micros=10_000_000, subscription_usd="10.00",
subscription_limit_micros=20_000_000, subscription_limit_usd="20.00",
purchased_micros=12_340_000, purchased_usd="12.34",
denominator_kind="subscription_cap", paid_access=True,
))
assert snap is not None and snap.provider == "nous"
win = next(w for w in snap.windows if w.label == "Subscription")
assert win.used_percent is not None and abs(win.used_percent - 50.0) < 1e-9
assert win.detail == "$10.00 of $20.00 left" # verbatim *_usd strings, not math
details = list(snap.details)
assert "Subscription credits: $10.00" in details
assert "Top-up credits: $12.34" in details
assert "Total usable: $30.34" in details
assert any("dev fixture" in d for d in details) # the offline marker
assert all("access depleted" not in d for d in details)
def test_depleted_adds_status_line():
snap = _snapshot_from_credits_state(_state(
remaining_micros=0, remaining_usd="0.00",
subscription_micros=0, subscription_usd="0.00",
purchased_micros=0, purchased_usd="0.00",
denominator_kind="none", paid_access=False,
))
assert snap is not None
assert any("access depleted" in d for d in snap.details)
def test_no_cap_yields_no_gauge_window():
# No subscription cap → used_fraction is None → no gauge window, magnitudes only.
snap = _snapshot_from_credits_state(_state(
remaining_micros=5_000_000, remaining_usd="5.00",
subscription_micros=5_000_000, subscription_usd="5.00",
subscription_limit_micros=None, denominator_kind="none", paid_access=True,
))
assert snap is not None
assert all(w.label != "Subscription" for w in snap.windows)
assert "Total usable: $5.00" in snap.details
def test_none_state_is_safe():
assert _snapshot_from_credits_state(None) is None

629
tests/agent/test_credits_policy.py Normal file
View file

@ -0,0 +1,629 @@
"""Tests for evaluate_credits_notices — pure threshold reconciliation policy (L4.1).
All tests use fresh latch = {"active": set(), "seen_below_90": False, "usage_band": None} per scenario.
CreditsState is constructed directly (not parsed from headers).
"""
from __future__ import annotations
import pytest
from agent.credits_tracker import (
CREDITS_NOTICE_KIND,
CREDITS_RESTORED_TTL_MS,
AgentNotice,
CreditsState,
evaluate_credits_notices,
)
# ── Helpers ──────────────────────────────────────────────────────────────────
def fresh_latch() -> dict:
return {"active": set(), "seen_below_90": False, "usage_band": None}
def state_with_fraction(
uf: float | None,
*,
paid_access: bool = True,
denominator_kind: str = "subscription_cap",
purchased_micros: int = 0,
purchased_usd: str = "0.00",
subscription_limit_usd: str | None = "20.00",
) -> CreditsState:
"""Build a minimal CreditsState that yields the desired used_fraction.
used_fraction = (limit - subscription_micros) / limit
When uf is None, we set limit to None so used_fraction returns None.
"""
if uf is None:
return CreditsState(
subscription_limit_micros=None,
subscription_limit_usd=None,
subscription_micros=0,
denominator_kind="none",
paid_access=paid_access,
purchased_micros=purchased_micros,
purchased_usd=purchased_usd,
)
# We want (limit - sub) / limit == uf → sub = limit * (1 - uf)
limit = 20_000_000 # $20 in micros
sub = int(limit * (1.0 - uf))
return CreditsState(
subscription_limit_micros=limit,
subscription_limit_usd=subscription_limit_usd,
subscription_micros=sub,
denominator_kind=denominator_kind,
paid_access=paid_access,
purchased_micros=purchased_micros,
purchased_usd=purchased_usd,
)
# ── Scenario 1: crossing 90% threshold ───────────────────────────────────────
class TestWarn90Crossing:
def test_below_lowest_band_no_notice_but_latch_set(self):
latch = fresh_latch()
s = state_with_fraction(0.10) # below the 50% band
to_show, to_clear = evaluate_credits_notices(s, latch)
assert all(n.key != "credits.usage" for n in to_show)
assert "credits.usage" not in to_clear
assert latch["seen_below_90"] is True
def test_crossing_to_90_fires_once(self):
latch = fresh_latch()
# First call: uf < 0.5 — sets seen_below_90 (below lowest band)
s1 = state_with_fraction(0.10)
evaluate_credits_notices(s1, latch)
# Second call: uf >= 0.9 — should fire the usage band at 90
s2 = state_with_fraction(0.95)
to_show, to_clear = evaluate_credits_notices(s2, latch)
keys = [n.key for n in to_show]
assert "credits.usage" in keys
assert "credits.usage" not in to_clear
def test_no_refire_on_repeated_over_90(self):
latch = fresh_latch()
s_below = state_with_fraction(0.10)
evaluate_credits_notices(s_below, latch)
s_over = state_with_fraction(0.95)
evaluate_credits_notices(s_over, latch)
# Third call: still ≥ 0.9 — must NOT re-fire
to_show, to_clear = evaluate_credits_notices(s_over, latch)
assert all(n.key != "credits.usage" for n in to_show)
assert "credits.usage" not in to_clear
# ── Scenario 2: recovery + re-cross ──────────────────────────────────────────
class TestWarn90RecoveryReCross:
def test_recovery_clears_warn90(self):
latch = fresh_latch()
# Cross below → above
evaluate_credits_notices(state_with_fraction(0.10), latch)
evaluate_credits_notices(state_with_fraction(0.95), latch)
# Recovery: uf drops back below ALL bands → usage notice clears entirely
to_show, to_clear = evaluate_credits_notices(state_with_fraction(0.10), latch)
assert "credits.usage" in to_clear
assert "credits.usage" not in latch["active"]
def test_recross_after_recovery_fires_again(self):
latch = fresh_latch()
evaluate_credits_notices(state_with_fraction(0.10), latch)
evaluate_credits_notices(state_with_fraction(0.95), latch)
evaluate_credits_notices(state_with_fraction(0.10), latch) # recovery
# Re-cross: uf >= 0.9 again — should fire again because the band is clearable
to_show, to_clear = evaluate_credits_notices(state_with_fraction(0.95), latch)
keys = [n.key for n in to_show]
assert "credits.usage" in keys
# ── Scenario 3: open-already-over (hybrid Q3 gate) ───────────────────────────
class TestOpenAlreadyOver:
def test_warn90_does_not_fire_without_seen_below_90(self):
"""First call uf≥0.9 with seen_below_90=False — warn90 must NOT fire."""
latch = fresh_latch()
assert latch["seen_below_90"] is False
s = state_with_fraction(0.95)
to_show, to_clear = evaluate_credits_notices(s, latch)
assert all(n.key != "credits.usage" for n in to_show)
assert "credits.usage" not in to_clear
# ── Scenario 3b: boundary — exact 0.9 and just-below-1.0 ────────────────────
class TestBoundaryFractions:
def test_exact_0_9_fires_warn90(self):
"""used_fraction == 0.9 exactly must fire warn90 (threshold is inclusive)."""
latch = fresh_latch()
# First: prime seen_below_90 with a sub-50% observation
evaluate_credits_notices(state_with_fraction(0.10), latch)
# Now construct a state where used_fraction is EXACTLY 0.9:
# subscription_limit_micros=20_000_000, subscription_micros=2_000_000
# → used = 18_000_000 / 20_000_000 = 0.9 exactly
s = CreditsState(
subscription_limit_micros=20_000_000,
subscription_limit_usd="20.00",
subscription_micros=2_000_000,
denominator_kind="subscription_cap",
paid_access=True,
)
assert s.used_fraction == 0.9
to_show, to_clear = evaluate_credits_notices(s, latch)
keys = [n.key for n in to_show]
assert "credits.usage" in keys
assert "credits.usage" not in to_clear
def test_just_below_1_0_does_not_fire_grant_spent(self):
"""subscription_micros = limit - 1 (used_fraction just under 1.0) must NOT fire grant_spent.
Locks the boundary so a future used_fraction clamp refactor cannot fire
grant_spent a micro early.
"""
latch = fresh_latch()
limit = 20_000_000
s = CreditsState(
subscription_limit_micros=limit,
subscription_limit_usd="20.00",
subscription_micros=1, # limit - 1 → used_fraction < 1.0
denominator_kind="subscription_cap",
purchased_micros=5_000_000,
purchased_usd="5.00",
paid_access=True,
)
assert s.used_fraction is not None and s.used_fraction < 1.0
to_show, to_clear = evaluate_credits_notices(s, latch)
assert all(n.key != "credits.grant_spent" for n in to_show)
assert "credits.grant_spent" not in to_clear
# ── Scenario 4: grant_spent ───────────────────────────────────────────────────
class TestGrantSpent:
def _grant_state(self, purchased_micros: int = 12_340_000) -> CreditsState:
return state_with_fraction(
1.0,
denominator_kind="subscription_cap",
purchased_micros=purchased_micros,
purchased_usd="12.34",
)
def test_grant_spent_fires_on_first_obs(self):
"""No crossing gate for grant_spent — fires immediately on first obs."""
latch = fresh_latch()
to_show, to_clear = evaluate_credits_notices(self._grant_state(), latch)
keys = [n.key for n in to_show]
assert "credits.grant_spent" in keys
def test_grant_spent_no_refire(self):
latch = fresh_latch()
evaluate_credits_notices(self._grant_state(), latch)
to_show, to_clear = evaluate_credits_notices(self._grant_state(), latch)
assert all(n.key != "credits.grant_spent" for n in to_show)
assert "credits.grant_spent" not in to_clear
def test_grant_spent_clears_when_purchased_zero(self):
latch = fresh_latch()
evaluate_credits_notices(self._grant_state(), latch)
# Now purchased → 0: grant_cond becomes False
s_no_purchase = state_with_fraction(
1.0,
denominator_kind="subscription_cap",
purchased_micros=0,
purchased_usd="0.00",
)
to_show, to_clear = evaluate_credits_notices(s_no_purchase, latch)
assert "credits.grant_spent" in to_clear
assert all(n.key != "credits.grant_spent" for n in to_show)
# ── Scenario 5: depleted + recovery ──────────────────────────────────────────
class TestDepleted:
def test_depleted_fires_level_error_kind_sticky(self):
latch = fresh_latch()
s = CreditsState(paid_access=False)
to_show, to_clear = evaluate_credits_notices(s, latch)
depleted_notices = [n for n in to_show if n.key == "credits.depleted"]
assert len(depleted_notices) == 1
n = depleted_notices[0]
assert n.level == "error"
assert n.kind == CREDITS_NOTICE_KIND
def test_recovery_emits_clear_and_restored(self):
latch = fresh_latch()
# Fire depleted
evaluate_credits_notices(CreditsState(paid_access=False), latch)
# Now recovered
to_show, to_clear = evaluate_credits_notices(CreditsState(paid_access=True), latch)
assert "credits.depleted" in to_clear
restored = [n for n in to_show if n.key == "credits.restored"]
assert len(restored) == 1
r = restored[0]
assert r.level == "success"
assert r.kind == "ttl"
assert r.ttl_ms == CREDITS_RESTORED_TTL_MS
def test_depleted_refires_after_recovery(self):
latch = fresh_latch()
evaluate_credits_notices(CreditsState(paid_access=False), latch)
evaluate_credits_notices(CreditsState(paid_access=True), latch)
# Goes depleted again
to_show, to_clear = evaluate_credits_notices(CreditsState(paid_access=False), latch)
keys = [n.key for n in to_show]
assert "credits.depleted" in keys
# ── Scenario 5b: free-model suppression of the depleted notice ───────────────
class TestDepletedFreeModelSuppression:
def test_depleted_suppressed_when_model_is_free(self):
latch = fresh_latch()
s = CreditsState(paid_access=False)
to_show, to_clear = evaluate_credits_notices(s, latch, model_is_free=True)
assert all(n.key != "credits.depleted" for n in to_show)
assert "credits.depleted" not in latch["active"]
assert to_clear == []
def test_switch_to_free_model_clears_without_restored(self):
latch = fresh_latch()
# Depleted on a paid model → notice fires
evaluate_credits_notices(CreditsState(paid_access=False), latch)
assert "credits.depleted" in latch["active"]
# Same depleted account, but now on a free model → clear, NO "restored"
to_show, to_clear = evaluate_credits_notices(
CreditsState(paid_access=False), latch, model_is_free=True
)
assert "credits.depleted" in to_clear
assert "credits.depleted" not in latch["active"]
assert all(n.key != "credits.restored" for n in to_show)
def test_switch_back_to_paid_model_while_depleted_reshows(self):
latch = fresh_latch()
evaluate_credits_notices(CreditsState(paid_access=False), latch)
evaluate_credits_notices(CreditsState(paid_access=False), latch, model_is_free=True)
# Back on a paid model, still depleted → notice re-fires
to_show, to_clear = evaluate_credits_notices(CreditsState(paid_access=False), latch)
keys = [n.key for n in to_show]
assert "credits.depleted" in keys
assert "credits.depleted" in latch["active"]
def test_genuine_recovery_on_free_model_no_spurious_restored(self):
"""Recovery observed while suppressed (notice never shown) → nothing to
clear, no 'restored' (there was no visible depleted state to restore)."""
latch = fresh_latch()
evaluate_credits_notices(CreditsState(paid_access=False), latch, model_is_free=True)
to_show, to_clear = evaluate_credits_notices(
CreditsState(paid_access=True), latch, model_is_free=True
)
assert to_clear == []
assert all(n.key != "credits.restored" for n in to_show)
def test_genuine_recovery_still_emits_restored_when_notice_active(self):
"""paid_access flip back to True with the notice showing → clear + restored
(unchanged behaviour, regardless of the model-free flag)."""
latch = fresh_latch()
evaluate_credits_notices(CreditsState(paid_access=False), latch)
to_show, to_clear = evaluate_credits_notices(
CreditsState(paid_access=True), latch, model_is_free=True
)
assert "credits.depleted" in to_clear
restored = [n for n in to_show if n.key == "credits.restored"]
assert len(restored) == 1
def test_free_flag_does_not_affect_other_notices(self):
"""Usage-band and grant notices are independent of the model-free gate."""
latch = fresh_latch()
evaluate_credits_notices(state_with_fraction(0.10), latch, model_is_free=True)
to_show, _ = evaluate_credits_notices(
state_with_fraction(0.95, paid_access=False), latch, model_is_free=True
)
keys = [n.key for n in to_show]
assert "credits.usage" in keys
assert "credits.depleted" not in keys
# ── Scenario 5c: is_free_tier_model (local-data-only check) ──────────────────
class TestIsFreeTierModel:
def test_free_suffix_is_free(self):
from agent.credits_tracker import is_free_tier_model
assert is_free_tier_model("nvidia/nemotron-3-ultra:free") is True
assert is_free_tier_model("Hermes-4-70B:free", "https://inference-api.nousresearch.com") is True
def test_empty_or_paid_model_is_not_free(self):
from agent.credits_tracker import is_free_tier_model
assert is_free_tier_model("") is False
assert is_free_tier_model("Hermes-4-405B") is False
def test_pricing_cache_peek_zero_priced_model(self, monkeypatch):
from agent.credits_tracker import is_free_tier_model
import hermes_cli.models as models_mod
# The picker keys the cache on the pre-/v1 root (get_pricing_for_provider
# strips a trailing /v1 before fetch_models_with_pricing).
monkeypatch.setattr(
models_mod,
"_pricing_cache",
{
"https://inference-api.nousresearch.com": {
"some/zero-priced": {"prompt": "0", "completion": "0"},
"some/paid": {"prompt": "0.000001", "completion": "0.000002"},
}
},
)
# The agent holds the /v1-suffixed URL (DEFAULT_NOUS_INFERENCE_URL) —
# the helper must normalize it down to the picker's cache key.
base = "https://inference-api.nousresearch.com/v1"
assert is_free_tier_model("some/zero-priced", base) is True
assert is_free_tier_model("some/paid", base) is False
# Pre-stripped and trailing-slash variants resolve to the same key.
assert is_free_tier_model("some/zero-priced", "https://inference-api.nousresearch.com/") is True
assert is_free_tier_model("some/zero-priced", "https://inference-api.nousresearch.com/v1/") is True
def test_cache_miss_is_not_free_and_no_fetch(self, monkeypatch):
from agent.credits_tracker import is_free_tier_model
import hermes_cli.models as models_mod
monkeypatch.setattr(models_mod, "_pricing_cache", {})
def _boom(*args, **kwargs): # any network attempt fails the test
raise AssertionError("is_free_tier_model must never hit the network")
import urllib.request
monkeypatch.setattr(urllib.request, "urlopen", _boom)
assert is_free_tier_model("some/model", "https://inference-api.nousresearch.com/v1") is False
def test_exception_fails_open_to_false(self, monkeypatch):
from agent.credits_tracker import is_free_tier_model
import hermes_cli.models as models_mod
class _Exploding:
def get(self, *_a, **_kw):
raise RuntimeError("boom")
monkeypatch.setattr(models_mod, "_pricing_cache", _Exploding())
assert is_free_tier_model("some/model", "https://inference-api.nousresearch.com") is False
# ── Scenario 6: denominator none (uf is None) ────────────────────────────────
class TestDenominatorNone:
def test_no_warn90_when_uf_none(self):
latch = fresh_latch()
s = state_with_fraction(None)
to_show, to_clear = evaluate_credits_notices(s, latch)
assert all(n.key != "credits.usage" for n in to_show)
assert "credits.usage" not in to_clear
def test_no_grant_spent_when_uf_none(self):
latch = fresh_latch()
s = CreditsState(
subscription_limit_micros=None,
denominator_kind="none",
purchased_micros=5_000_000,
purchased_usd="5.00",
)
to_show, to_clear = evaluate_credits_notices(s, latch)
assert all(n.key != "credits.grant_spent" for n in to_show)
def test_warn90_clears_when_uf_becomes_none(self):
"""If warn90 was active and uf becomes None, it should clear."""
latch = fresh_latch()
# Establish usage notice active: cross below → above
evaluate_credits_notices(state_with_fraction(0.10), latch)
evaluate_credits_notices(state_with_fraction(0.95), latch)
assert "credits.usage" in latch["active"]
# Now uf becomes None (denominator changed to "none")
s_none = state_with_fraction(None)
to_show, to_clear = evaluate_credits_notices(s_none, latch)
assert "credits.usage" in to_clear
assert "credits.usage" not in latch["active"]
# ── Scenario 7: copy / verbatim USD strings ──────────────────────────────────
class TestNoticeCopy:
def test_warn90_contains_verbatim_subscription_limit_usd(self):
latch = fresh_latch()
evaluate_credits_notices(state_with_fraction(0.10), latch)
s = state_with_fraction(0.95, subscription_limit_usd="20.00")
to_show, _ = evaluate_credits_notices(s, latch)
warn_notice = next(n for n in to_show if n.key == "credits.usage")
assert "$20.00" in warn_notice.text
assert "cap" in warn_notice.text
def test_grant_spent_contains_verbatim_purchased_usd(self):
latch = fresh_latch()
s = state_with_fraction(
1.0,
denominator_kind="subscription_cap",
purchased_micros=12_340_000,
purchased_usd="12.34",
)
to_show, _ = evaluate_credits_notices(s, latch)
grant_notice = next(n for n in to_show if n.key == "credits.grant_spent")
assert "$12.34" in grant_notice.text
assert "top-up left" in grant_notice.text
def test_depleted_mentions_usage_command(self):
latch = fresh_latch()
s = CreditsState(paid_access=False)
to_show, _ = evaluate_credits_notices(s, latch)
depleted_notice = next(n for n in to_show if n.key == "credits.depleted")
assert "/usage" in depleted_notice.text
# ── Scenario 8: severity order in a single call ──────────────────────────────
class TestSeverityOrder:
def test_multiple_new_notices_ordered_ascending_severity(self):
"""warn90 < grant_spent < depleted in to_show when all fire in one call."""
# Construct a state where all three conditions fire simultaneously
# on first call (no latch state yet):
# - warn90: uf >= 0.9 AND seen_below_90 must be True → won't fire fresh latch
# So we pre-seed seen_below_90=True to allow warn90 to fire.
latch = {"active": set(), "seen_below_90": True, "usage_band": None}
# Build state: subscription_cap, uf >= 1.0, purchased_micros > 0, NOT paid_access
# warn90_cond: uf >= 0.9 ✓ (uf=1.0)
# grant_cond: subscription_cap + uf >= 1.0 + purchased > 0 ✓
# depleted_cond: not paid_access ✓
s = CreditsState(
subscription_limit_micros=20_000_000,
subscription_limit_usd="20.00",
subscription_micros=0, # uf = 1.0
denominator_kind="subscription_cap",
purchased_micros=5_000_000,
purchased_usd="5.00",
paid_access=False,
)
to_show, _ = evaluate_credits_notices(s, latch)
keys = [n.key for n in to_show]
assert "credits.usage" in keys
assert "credits.grant_spent" in keys
assert "credits.depleted" in keys
# Ascending severity: warn90 before grant_spent before depleted
assert keys.index("credits.usage") < keys.index("credits.grant_spent")
assert keys.index("credits.grant_spent") < keys.index("credits.depleted")
# ── Invariant: never fire + clear same key in one call ────────────────────────
class TestNoFireAndClearSameKey:
def test_usage_never_both_fired_and_cleared(self):
latch = fresh_latch()
# Run many state transitions; across each, assert no key is in both lists
states = [
state_with_fraction(0.10),
state_with_fraction(0.95),
state_with_fraction(0.10),
state_with_fraction(0.95),
state_with_fraction(None),
]
for s in states:
to_show, to_clear = evaluate_credits_notices(s, latch)
fired_keys = {n.key for n in to_show}
cleared_keys = set(to_clear)
overlap = fired_keys & cleared_keys
assert not overlap, f"Key(s) both fired and cleared: {overlap}"
def test_depleted_never_both_fired_and_cleared(self):
latch = fresh_latch()
states = [
CreditsState(paid_access=False),
CreditsState(paid_access=True),
CreditsState(paid_access=False),
]
for s in states:
to_show, to_clear = evaluate_credits_notices(s, latch)
fired_keys = {n.key for n in to_show}
cleared_keys = set(to_clear)
overlap = fired_keys & cleared_keys
assert not overlap, f"Key(s) both fired and cleared: {overlap}"
# ── Scenario 9: escalating usage bands (50 → 75 → 90) ────────────────────────
class TestUsageBands:
"""The usage notice shows the HIGHEST crossed band as a single escalating line."""
def _band_text(self, to_show):
n = next((n for n in to_show if n.key == "credits.usage"), None)
return n.text if n else None
def test_50_band_fires_info(self):
latch = fresh_latch()
evaluate_credits_notices(state_with_fraction(0.10), latch) # prime
to_show, _ = evaluate_credits_notices(state_with_fraction(0.55), latch)
n = next(n for n in to_show if n.key == "credits.usage")
assert "50%" in n.text and n.level == "info"
assert latch["usage_band"] == 50
def test_75_band_fires_warn(self):
latch = fresh_latch()
evaluate_credits_notices(state_with_fraction(0.10), latch)
to_show, _ = evaluate_credits_notices(state_with_fraction(0.80), latch)
n = next(n for n in to_show if n.key == "credits.usage")
assert "75%" in n.text and n.level == "warn"
assert latch["usage_band"] == 75
def test_climb_replaces_band(self):
"""Climbing 50→75→90 replaces the single line (clear old + show new)."""
latch = fresh_latch()
evaluate_credits_notices(state_with_fraction(0.10), latch)
# 55% → 50 band
evaluate_credits_notices(state_with_fraction(0.55), latch)
assert latch["usage_band"] == 50
# 80% → climbs to 75, clearing the 50 line
to_show, to_clear = evaluate_credits_notices(state_with_fraction(0.80), latch)
assert "credits.usage" in to_clear
assert "75%" in self._band_text(to_show)
assert latch["usage_band"] == 75
# 95% → climbs to 90
to_show, to_clear = evaluate_credits_notices(state_with_fraction(0.95), latch)
assert "credits.usage" in to_clear
assert "90%" in self._band_text(to_show)
assert latch["usage_band"] == 90
def test_step_down_on_recovery(self):
"""Recovering steps the band back down, then clears below the lowest band."""
latch = fresh_latch()
evaluate_credits_notices(state_with_fraction(0.10), latch)
evaluate_credits_notices(state_with_fraction(0.95), latch)
assert latch["usage_band"] == 90
# drop to 80% → steps down to 75
to_show, to_clear = evaluate_credits_notices(state_with_fraction(0.80), latch)
assert "credits.usage" in to_clear
assert "75%" in self._band_text(to_show)
# drop to 55% → steps down to 50
to_show, _ = evaluate_credits_notices(state_with_fraction(0.55), latch)
assert "50%" in self._band_text(to_show)
# drop below 50% → clears entirely
to_show, to_clear = evaluate_credits_notices(state_with_fraction(0.10), latch)
assert "credits.usage" in to_clear
assert latch["usage_band"] is None
def test_no_refire_same_band(self):
latch = fresh_latch()
evaluate_credits_notices(state_with_fraction(0.10), latch)
evaluate_credits_notices(state_with_fraction(0.80), latch) # fires 75
# still 80% → same band, no re-emit, no clear
to_show, to_clear = evaluate_credits_notices(state_with_fraction(0.80), latch)
assert all(n.key != "credits.usage" for n in to_show)
assert "credits.usage" not in to_clear
def test_exact_band_boundaries_inclusive(self):
"""Thresholds are inclusive: exactly 0.50 / 0.75 / 0.90 land in their band."""
for uf, want in [(0.50, 50), (0.75, 75), (0.90, 90)]:
latch = fresh_latch()
latch["seen_below_90"] = True # allow firing
evaluate_credits_notices(state_with_fraction(uf), latch)
assert latch["usage_band"] == want, (uf, latch["usage_band"])
def test_open_below_lowest_band_no_notice(self):
latch = fresh_latch()
to_show, to_clear = evaluate_credits_notices(state_with_fraction(0.30), latch)
assert all(n.key != "credits.usage" for n in to_show)
assert latch["usage_band"] is None

909
tests/agent/test_credits_tracker.py Normal file
View file

@ -0,0 +1,909 @@
"""Tests for agent.credits_tracker — CreditsState + parse_credits_headers.
Covers the 9-state matrix plus validation edge cases. All header values
arrive as STRINGS (the producer calls String(...) on every field).
"""
from __future__ import annotations
import logging
import time
from typing import Optional
import pytest
from agent.credits_tracker import CreditsState, parse_credits_headers
# ── Helpers ─────────────────────────────────────────────────────────────────
def micros(dollars: float) -> str:
"""Convert a dollar amount to a micros string for header fixtures."""
return str(round(dollars * 1_000_000))
# ── 9-State matrix fixtures ──────────────────────────────────────────────────
def _base_headers(**overrides) -> dict:
"""Base headers present in every valid response."""
h = {
"x-nous-credits-version": "1",
"x-nous-credits-remaining-micros": micros(0),
"x-nous-credits-remaining-usd": "0.00",
"x-nous-credits-subscription-micros": micros(0),
"x-nous-credits-subscription-usd": "0.00",
"x-nous-credits-rollover-micros": micros(0),
"x-nous-credits-purchased-micros": micros(0),
"x-nous-credits-purchased-usd": "0.00",
"x-nous-tool-pool-micros": micros(0),
"x-nous-tool-pool-gated-off": "false",
"x-nous-credits-denominator-kind": "none",
"x-nous-credits-paid-access": "true",
"x-nous-credits-as-of-ms": "1717000000000",
}
h.update(overrides)
return h
# ── 9 STATES ────────────────────────────────────────────────────────────────
HEALTHY_HEADERS = _base_headers(
**{
"x-nous-credits-remaining-micros": micros(30.34),
"x-nous-credits-remaining-usd": "30.34",
"x-nous-credits-subscription-micros": micros(18.00),
"x-nous-credits-subscription-usd": "18.00",
"x-nous-credits-subscription-limit-micros": micros(20.00),
"x-nous-credits-subscription-limit-usd": "20.00",
"x-nous-credits-rollover-micros": micros(0),
"x-nous-credits-purchased-micros": micros(12.34),
"x-nous-credits-purchased-usd": "12.34",
"x-nous-tool-pool-micros": micros(2.00),
"x-nous-tool-pool-gated-off": "true",
"x-nous-credits-denominator-kind": "subscription_cap",
"x-nous-credits-paid-access": "true",
}
)
SUB_90PCT_HEADERS = _base_headers(
**{
"x-nous-credits-remaining-micros": micros(2.00),
"x-nous-credits-remaining-usd": "2.00",
"x-nous-credits-subscription-micros": micros(2.00),
"x-nous-credits-subscription-usd": "2.00",
"x-nous-credits-subscription-limit-micros": micros(20.00),
"x-nous-credits-subscription-limit-usd": "20.00",
"x-nous-credits-purchased-micros": micros(0),
"x-nous-credits-purchased-usd": "0.00",
"x-nous-credits-denominator-kind": "subscription_cap",
"x-nous-credits-paid-access": "true",
}
)
GRANT_EXHAUSTED_HEADERS = _base_headers(
**{
"x-nous-credits-remaining-micros": micros(12.34),
"x-nous-credits-remaining-usd": "12.34",
"x-nous-credits-subscription-micros": micros(0),
"x-nous-credits-subscription-usd": "0.00",
"x-nous-credits-subscription-limit-micros": micros(20.00),
"x-nous-credits-subscription-limit-usd": "20.00",
"x-nous-credits-purchased-micros": micros(12.34),
"x-nous-credits-purchased-usd": "12.34",
"x-nous-credits-denominator-kind": "subscription_cap",
"x-nous-credits-paid-access": "true",
}
)
PURCHASED_ONLY_HEADERS = _base_headers(
**{
"x-nous-credits-remaining-micros": micros(30.00),
"x-nous-credits-remaining-usd": "30.00",
"x-nous-credits-subscription-micros": micros(0),
"x-nous-credits-subscription-usd": "0.00",
"x-nous-credits-purchased-micros": micros(30.00),
"x-nous-credits-purchased-usd": "30.00",
"x-nous-credits-denominator-kind": "none",
"x-nous-credits-paid-access": "true",
# No limit pair — denominator_kind=none
}
)
TOOL_POOL_FREE_HEADERS = _base_headers(
**{
"x-nous-credits-remaining-micros": micros(0.05),
"x-nous-credits-remaining-usd": "0.05",
"x-nous-tool-pool-micros": micros(0.05),
"x-nous-tool-pool-gated-off": "false",
"x-nous-credits-paid-access": "true",
}
)
DEPLETED_HEADERS = _base_headers(
**{
"x-nous-credits-remaining-micros": micros(0),
"x-nous-credits-remaining-usd": "0.00",
"x-nous-credits-subscription-micros": micros(0),
"x-nous-credits-subscription-usd": "0.00",
"x-nous-credits-purchased-micros": micros(0),
"x-nous-credits-purchased-usd": "0.00",
"x-nous-credits-paid-access": "false",
"x-nous-credits-disabled-reason": "out_of_credits",
}
)
DEBT_HEADERS = _base_headers(
**{
"x-nous-credits-remaining-micros": micros(0),
"x-nous-credits-remaining-usd": "0.00",
"x-nous-credits-subscription-micros": str(-5_000_000),
"x-nous-credits-subscription-usd": "-5.00",
"x-nous-credits-purchased-micros": micros(0),
"x-nous-credits-purchased-usd": "0.00",
"x-nous-credits-paid-access": "false",
}
)
# ── State 1: healthy ─────────────────────────────────────────────────────────
class TestHealthyState:
def test_parses_successfully(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state is not None
def test_from_header_set(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state.from_header is True
def test_captured_at_set(self):
before = time.time()
state = parse_credits_headers(HEALTHY_HEADERS)
after = time.time()
assert before <= state.captured_at <= after
def test_remaining_fields(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state.remaining_micros == round(30.34 * 1_000_000)
assert state.remaining_usd == "30.34"
def test_subscription_fields(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state.subscription_micros == round(18.00 * 1_000_000)
assert state.subscription_usd == "18.00"
assert state.subscription_limit_micros == round(20.00 * 1_000_000)
assert state.subscription_limit_usd == "20.00"
def test_rollover_and_purchased(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state.rollover_micros == 0
assert state.purchased_micros == round(12.34 * 1_000_000)
assert state.purchased_usd == "12.34"
def test_tool_pool(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state.tool_pool_micros == round(2.00 * 1_000_000)
assert state.tool_pool_gated_off is True
def test_denominator_and_access(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state.denominator_kind == "subscription_cap"
assert state.paid_access is True
assert state.disabled_reason is None
def test_used_fraction(self):
state = parse_credits_headers(HEALTHY_HEADERS)
# (20.00 - 18.00) / 20.00 = 0.10
assert state.used_fraction == pytest.approx(0.10)
def test_has_data(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state.has_data is True
def test_not_depleted(self):
state = parse_credits_headers(HEALTHY_HEADERS)
assert state.depleted is False
def test_age_seconds_reasonable(self):
state = parse_credits_headers(HEALTHY_HEADERS)
# Should be very small — just parsed
assert 0 <= state.age_seconds < 5
# ── State 2: sub_90pct ───────────────────────────────────────────────────────
class TestSub90Pct:
def test_parses_successfully(self):
state = parse_credits_headers(SUB_90PCT_HEADERS)
assert state is not None
def test_used_fraction_90pct(self):
state = parse_credits_headers(SUB_90PCT_HEADERS)
# (20.00 - 2.00) / 20.00 = 0.90
assert state.used_fraction == pytest.approx(0.90)
def test_paid_access(self):
state = parse_credits_headers(SUB_90PCT_HEADERS)
assert state.paid_access is True
assert state.depleted is False
# ── State 3: grant_exhausted ─────────────────────────────────────────────────
class TestGrantExhausted:
def test_used_fraction_100pct(self):
state = parse_credits_headers(GRANT_EXHAUSTED_HEADERS)
assert state is not None
# subscription_micros=0, limit=20.00 → (20-0)/20 = 1.0
assert state.used_fraction == pytest.approx(1.0)
def test_paid_access_still_true(self):
state = parse_credits_headers(GRANT_EXHAUSTED_HEADERS)
assert state.paid_access is True
assert state.depleted is False
# ── State 4: purchased_only ──────────────────────────────────────────────────
class TestPurchasedOnly:
def test_parses_successfully(self):
state = parse_credits_headers(PURCHASED_ONLY_HEADERS)
assert state is not None
def test_denominator_kind_none(self):
state = parse_credits_headers(PURCHASED_ONLY_HEADERS)
assert state.denominator_kind == "none"
def test_used_fraction_is_none_no_limit(self):
state = parse_credits_headers(PURCHASED_ONLY_HEADERS)
# No subscription_limit_micros → used_fraction is None
assert state.used_fraction is None
def test_no_limit_pair(self):
state = parse_credits_headers(PURCHASED_ONLY_HEADERS)
assert state.subscription_limit_micros is None
assert state.subscription_limit_usd is None
# ── State 5: tool_pool_free ──────────────────────────────────────────────────
class TestToolPoolFree:
def test_parses_successfully(self):
state = parse_credits_headers(TOOL_POOL_FREE_HEADERS)
assert state is not None
def test_tool_pool_gated_off_false(self):
state = parse_credits_headers(TOOL_POOL_FREE_HEADERS)
assert state.tool_pool_gated_off is False
def test_tool_pool_micros(self):
state = parse_credits_headers(TOOL_POOL_FREE_HEADERS)
assert state.tool_pool_micros == round(0.05 * 1_000_000)
def test_paid_access(self):
state = parse_credits_headers(TOOL_POOL_FREE_HEADERS)
assert state.paid_access is True
# ── State 6: depleted ────────────────────────────────────────────────────────
class TestDepleted:
def test_parses_successfully(self):
state = parse_credits_headers(DEPLETED_HEADERS)
assert state is not None
def test_paid_access_false(self):
state = parse_credits_headers(DEPLETED_HEADERS)
assert state.paid_access is False
def test_depleted_true(self):
state = parse_credits_headers(DEPLETED_HEADERS)
assert state.depleted is True
def test_disabled_reason(self):
state = parse_credits_headers(DEPLETED_HEADERS)
assert state.disabled_reason == "out_of_credits"
def test_remaining_zero(self):
state = parse_credits_headers(DEPLETED_HEADERS)
assert state.remaining_micros == 0
# ── State 7: debt ────────────────────────────────────────────────────────────
class TestDebt:
def test_parses_successfully(self):
# Negative subscription_micros should NOT cause the parse to fail
state = parse_credits_headers(DEBT_HEADERS)
assert state is not None
def test_negative_subscription_accepted(self):
state = parse_credits_headers(DEBT_HEADERS)
assert state.subscription_micros == -5_000_000
def test_negative_subscription_usd_accepted(self):
state = parse_credits_headers(DEBT_HEADERS)
assert state.subscription_usd == "-5.00"
def test_paid_access_false(self):
state = parse_credits_headers(DEBT_HEADERS)
assert state.paid_access is False
assert state.depleted is True
# ── State 8: missing ─────────────────────────────────────────────────────────
class TestMissing:
def test_no_credits_headers_returns_none(self):
state = parse_credits_headers({})
assert state is None
def test_completely_empty_dict(self):
assert parse_credits_headers({}) is None
# ── State 9: no_org ──────────────────────────────────────────────────────────
class TestNoOrg:
def test_irrelevant_headers_return_none(self):
headers = {
"content-type": "application/json",
"x-request-id": "abc123",
"server": "nginx",
}
state = parse_credits_headers(headers)
assert state is None
def test_api_key_path_no_org_returns_none(self):
# Headers that might appear on an api-key path with no org
headers = {
"content-type": "application/json",
"authorization": "Bearer sk-test",
}
assert parse_credits_headers(headers) is None
# ── Version validation ───────────────────────────────────────────────────────
class TestVersionValidation:
def test_version_string_1_parses(self):
headers = _base_headers(**{"x-nous-credits-version": "1"})
state = parse_credits_headers(headers)
assert state is not None
assert state.version == 1
def test_version_2_returns_none(self):
headers = _base_headers(**{"x-nous-credits-version": "2"})
state = parse_credits_headers(headers)
assert state is None
def test_version_absent_returns_none(self):
headers = {k: v for k, v in _base_headers().items() if k != "x-nous-credits-version"}
state = parse_credits_headers(headers)
assert state is None
def test_version_greater_than_1_warns_once(self, caplog):
"""Version > 1 must log a warning, and ONLY ONCE across multiple calls."""
import agent.credits_tracker as ct
original = ct._version_warning_emitted
try:
# Reset the warn-once latch so this test starts clean regardless of order
ct._version_warning_emitted = False
headers = _base_headers(**{"x-nous-credits-version": "3"})
with caplog.at_level(logging.WARNING, logger="agent.credits_tracker"):
parse_credits_headers(headers)
parse_credits_headers(headers)
parse_credits_headers(headers)
warning_records = [r for r in caplog.records if "unsupported" in r.message.lower() or "version" in r.message.lower()]
assert len(warning_records) == 1, (
f"Expected exactly 1 version warning, got {len(warning_records)}: {[r.message for r in warning_records]}"
)
finally:
ct._version_warning_emitted = original
def test_version_0_returns_none(self):
headers = _base_headers(**{"x-nous-credits-version": "0"})
assert parse_credits_headers(headers) is None
def test_version_non_int_returns_none(self):
headers = _base_headers(**{"x-nous-credits-version": "abc"})
assert parse_credits_headers(headers) is None
# ── Bool-string trap ─────────────────────────────────────────────────────────
class TestBoolStringTrap:
"""Explicit tests for the bool("false") == True trap."""
def test_paid_access_string_false_means_depleted(self):
"""paid_access='false' must yield paid_access=False — NOT True."""
headers = _base_headers(**{"x-nous-credits-paid-access": "false"})
state = parse_credits_headers(headers)
assert state is not None
assert state.paid_access is False
assert state.depleted is True
def test_paid_access_string_true_means_not_depleted(self):
headers = _base_headers(**{"x-nous-credits-paid-access": "true"})
state = parse_credits_headers(headers)
assert state is not None
assert state.paid_access is True
assert state.depleted is False
def test_paid_access_case_insensitive_FALSE(self):
headers = _base_headers(**{"x-nous-credits-paid-access": "FALSE"})
state = parse_credits_headers(headers)
assert state is not None
assert state.paid_access is False
def test_paid_access_case_insensitive_True(self):
headers = _base_headers(**{"x-nous-credits-paid-access": "True"})
state = parse_credits_headers(headers)
assert state is not None
assert state.paid_access is True
def test_tool_pool_gated_off_false(self):
headers = _base_headers(**{"x-nous-tool-pool-gated-off": "false"})
state = parse_credits_headers(headers)
assert state is not None
assert state.tool_pool_gated_off is False
def test_tool_pool_gated_off_true(self):
headers = _base_headers(**{"x-nous-tool-pool-gated-off": "true"})
state = parse_credits_headers(headers)
assert state is not None
assert state.tool_pool_gated_off is True
# ── Tool-pool optional headers ────────────────────────────────────────────────
class TestToolPoolOptional:
"""x-nous-tool-pool-* headers are optional; absent → defaults; present-but-malformed → miss."""
def _no_tool_pool_headers(self) -> dict:
"""Base headers with BOTH tool-pool headers removed."""
h = _base_headers()
h.pop("x-nous-tool-pool-micros", None)
h.pop("x-nous-tool-pool-gated-off", None)
return h
def test_absent_tool_pool_headers_parse_succeeds(self):
"""Valid credits headers with no x-nous-tool-pool-* → parse succeeds."""
state = parse_credits_headers(self._no_tool_pool_headers())
assert state is not None
def test_absent_tool_pool_micros_defaults_to_zero(self):
state = parse_credits_headers(self._no_tool_pool_headers())
assert state.tool_pool_micros == 0
def test_absent_tool_pool_gated_off_defaults_to_false(self):
state = parse_credits_headers(self._no_tool_pool_headers())
assert state.tool_pool_gated_off is False
def test_present_malformed_tool_pool_micros_returns_none(self):
"""x-nous-tool-pool-micros present but non-int → parse miss (returns None)."""
headers = _base_headers(**{"x-nous-tool-pool-micros": "not-a-number"})
assert parse_credits_headers(headers) is None
def test_present_negative_tool_pool_micros_returns_none(self):
"""x-nous-tool-pool-micros present but negative → parse miss (returns None)."""
headers = _base_headers(**{"x-nous-tool-pool-micros": "-1000"})
assert parse_credits_headers(headers) is None
def test_only_tool_pool_micros_absent_still_succeeds(self):
"""Only micros absent (gated-off still present) → tool_pool_micros = 0, parse succeeds."""
h = _base_headers()
h.pop("x-nous-tool-pool-micros", None)
state = parse_credits_headers(h)
assert state is not None
assert state.tool_pool_micros == 0
# ── Half-pair subscription limit ─────────────────────────────────────────────
class TestHalfPairLimit:
def test_only_limit_micros_present_both_absent(self):
"""Only -micros present → both None, parse SUCCEEDS."""
headers = _base_headers(
**{
"x-nous-credits-subscription-limit-micros": micros(20.00),
"x-nous-credits-denominator-kind": "subscription_cap",
}
)
state = parse_credits_headers(headers)
assert state is not None
assert state.subscription_limit_micros is None
assert state.subscription_limit_usd is None
def test_only_limit_usd_present_both_absent(self):
"""Only -usd present → both None, parse SUCCEEDS."""
headers = _base_headers(
**{
"x-nous-credits-subscription-limit-usd": "20.00",
"x-nous-credits-denominator-kind": "subscription_cap",
}
)
state = parse_credits_headers(headers)
assert state is not None
assert state.subscription_limit_micros is None
assert state.subscription_limit_usd is None
def test_half_pair_used_fraction_is_none(self):
"""With no limit pair, used_fraction is None regardless of denominator_kind."""
headers = _base_headers(
**{
"x-nous-credits-subscription-limit-micros": micros(20.00),
"x-nous-credits-denominator-kind": "subscription_cap",
}
)
state = parse_credits_headers(headers)
assert state is not None
assert state.used_fraction is None
def test_full_pair_present_parsed_correctly(self):
"""Both present → both populated, used_fraction computable."""
headers = _base_headers(
**{
"x-nous-credits-subscription-micros": micros(10.00),
"x-nous-credits-subscription-usd": "10.00",
"x-nous-credits-subscription-limit-micros": micros(20.00),
"x-nous-credits-subscription-limit-usd": "20.00",
"x-nous-credits-denominator-kind": "subscription_cap",
}
)
state = parse_credits_headers(headers)
assert state is not None
assert state.subscription_limit_micros == round(20.00 * 1_000_000)
assert state.subscription_limit_usd == "20.00"
assert state.used_fraction == pytest.approx(0.50)
# ── Negative value validation ─────────────────────────────────────────────────
class TestNegativeValues:
def test_negative_remaining_micros_returns_none(self):
headers = _base_headers(**{"x-nous-credits-remaining-micros": "-1000"})
assert parse_credits_headers(headers) is None
def test_negative_purchased_micros_returns_none(self):
headers = _base_headers(**{"x-nous-credits-purchased-micros": "-500"})
assert parse_credits_headers(headers) is None
def test_negative_rollover_micros_returns_none(self):
headers = _base_headers(**{"x-nous-credits-rollover-micros": "-100"})
assert parse_credits_headers(headers) is None
def test_negative_limit_micros_returns_none(self):
headers = _base_headers(
**{
"x-nous-credits-subscription-limit-micros": "-1000",
"x-nous-credits-subscription-limit-usd": "-0.00",
"x-nous-credits-denominator-kind": "subscription_cap",
}
)
assert parse_credits_headers(headers) is None
def test_negative_subscription_accepted(self):
"""subscription_micros is the ONLY field allowed to be negative."""
headers = _base_headers(**{"x-nous-credits-subscription-micros": "-5000000",
"x-nous-credits-subscription-usd": "-5.00"})
state = parse_credits_headers(headers)
assert state is not None
assert state.subscription_micros == -5_000_000
# ── USD format validation ─────────────────────────────────────────────────────
class TestUsdValidation:
def test_valid_usd_format(self):
headers = _base_headers(**{"x-nous-credits-remaining-usd": "18.00"})
state = parse_credits_headers(headers)
assert state is not None
assert state.remaining_usd == "18.00"
def test_usd_one_decimal_returns_none(self):
"""'18.0' does not match ^-?\d+\.\d{2}$"""
headers = _base_headers(**{"x-nous-credits-remaining-usd": "18.0"})
assert parse_credits_headers(headers) is None
def test_usd_no_decimal_returns_none(self):
headers = _base_headers(**{"x-nous-credits-remaining-usd": "18"})
assert parse_credits_headers(headers) is None
def test_usd_with_dollar_sign_returns_none(self):
headers = _base_headers(**{"x-nous-credits-remaining-usd": "$18.00"})
assert parse_credits_headers(headers) is None
def test_usd_with_comma_returns_none(self):
headers = _base_headers(**{"x-nous-credits-remaining-usd": "1,800.00"})
assert parse_credits_headers(headers) is None
def test_usd_negative_valid(self):
"""Negative USD string should parse (e.g. subscription debt)."""
headers = _base_headers(
**{
"x-nous-credits-subscription-micros": "-5000000",
"x-nous-credits-subscription-usd": "-5.00",
}
)
state = parse_credits_headers(headers)
assert state is not None
assert state.subscription_usd == "-5.00"
# ── Non-int micros validation ─────────────────────────────────────────────────
class TestMicrosValidation:
def test_non_int_micros_string_returns_none(self):
headers = _base_headers(**{"x-nous-credits-remaining-micros": "abc"})
assert parse_credits_headers(headers) is None
def test_float_string_micros_returns_none(self):
"""'1.5' is not an integer string — should fail validation."""
headers = _base_headers(**{"x-nous-credits-remaining-micros": "1.5"})
assert parse_credits_headers(headers) is None
def test_non_int_purchased_returns_none(self):
headers = _base_headers(**{"x-nous-credits-purchased-micros": "abc"})
assert parse_credits_headers(headers) is None
# ── as_of_ms validation ───────────────────────────────────────────────────────
class TestAsOfMs:
def test_junk_as_of_ms_returns_none(self):
headers = _base_headers(**{"x-nous-credits-as-of-ms": "not-a-timestamp"})
assert parse_credits_headers(headers) is None
def test_valid_as_of_ms(self):
headers = _base_headers(**{"x-nous-credits-as-of-ms": "1717000000000"})
state = parse_credits_headers(headers)
assert state is not None
assert state.as_of_ms == 1717000000000
# ── denominator_kind validation ────────────────────────────────────────────────
class TestDenominatorKind:
def test_subscription_cap_valid(self):
headers = _base_headers(
**{
"x-nous-credits-denominator-kind": "subscription_cap",
"x-nous-credits-subscription-limit-micros": micros(20.00),
"x-nous-credits-subscription-limit-usd": "20.00",
}
)
state = parse_credits_headers(headers)
assert state is not None
assert state.denominator_kind == "subscription_cap"
def test_none_valid(self):
headers = _base_headers(**{"x-nous-credits-denominator-kind": "none"})
state = parse_credits_headers(headers)
assert state is not None
assert state.denominator_kind == "none"
def test_invalid_denominator_kind_returns_none(self):
headers = _base_headers(**{"x-nous-credits-denominator-kind": "invalid_kind"})
assert parse_credits_headers(headers) is None
# ── Zero-division guard ────────────────────────────────────────────────────────
class TestZeroDivisionGuard:
def test_subscription_limit_zero_used_fraction_is_none(self):
"""subscription_limit_micros='0' + subscription_cap → used_fraction is None (no ZeroDivisionError)."""
headers = _base_headers(
**{
"x-nous-credits-subscription-limit-micros": "0",
"x-nous-credits-subscription-limit-usd": "0.00",
"x-nous-credits-denominator-kind": "subscription_cap",
}
)
state = parse_credits_headers(headers)
assert state is not None
# limit == 0, so used_fraction must be None (guard prevents division)
assert state.used_fraction is None
# ── Unknown headers ignored ────────────────────────────────────────────────────
class TestUnknownHeaders:
def test_unknown_extra_header_ignored(self):
headers = {
**_base_headers(),
"x-nous-credits-future-field": "some-value",
"x-request-id": "abc123",
}
state = parse_credits_headers(headers)
assert state is not None
def test_mixed_with_other_providers_headers(self):
headers = {
**_base_headers(),
"x-ratelimit-limit-requests": "800",
"content-type": "application/json",
}
state = parse_credits_headers(headers)
assert state is not None
# ── Header normalization ──────────────────────────────────────────────────────
class TestHeaderNormalization:
def test_uppercase_headers_parsed(self):
headers = {k.upper(): v for k, v in _base_headers().items()}
state = parse_credits_headers(headers)
assert state is not None
def test_mixed_case_headers_parsed(self):
headers = {
"X-Nous-Credits-Version": "1",
"X-Nous-Credits-Remaining-Micros": micros(5.00),
"X-Nous-Credits-Remaining-Usd": "5.00",
"X-Nous-Credits-Subscription-Micros": micros(5.00),
"X-Nous-Credits-Subscription-Usd": "5.00",
"X-Nous-Credits-Rollover-Micros": "0",
"X-Nous-Credits-Purchased-Micros": "0",
"X-Nous-Credits-Purchased-Usd": "0.00",
"X-Nous-Tool-Pool-Micros": "0",
"X-Nous-Tool-Pool-Gated-Off": "false",
"X-Nous-Credits-Denominator-Kind": "none",
"X-Nous-Credits-Paid-Access": "true",
"X-Nous-Credits-As-Of-Ms": "1717000000000",
}
state = parse_credits_headers(headers)
assert state is not None
assert state.remaining_micros == round(5.00 * 1_000_000)
# ── CreditsState dataclass defaults ──────────────────────────────────────────
class TestCreditsStateDefaults:
def test_default_state(self):
state = CreditsState()
assert state.version == 0
assert state.remaining_micros == 0
assert state.remaining_usd == ""
assert state.subscription_micros == 0
assert state.subscription_usd == ""
assert state.subscription_limit_micros is None
assert state.subscription_limit_usd is None
assert state.rollover_micros == 0
assert state.purchased_micros == 0
assert state.purchased_usd == ""
assert state.tool_pool_micros == 0
assert state.tool_pool_gated_off is False
assert state.denominator_kind == "none"
assert state.paid_access is True
assert state.disabled_reason is None
assert state.as_of_ms == 0
assert state.captured_at == 0.0
assert state.from_header is False
def test_has_data_false_when_no_captured_at(self):
state = CreditsState()
assert state.has_data is False
def test_age_seconds_inf_when_no_data(self):
state = CreditsState()
assert state.age_seconds == float("inf")
def test_depleted_false_by_default(self):
state = CreditsState()
assert state.depleted is False
def test_used_fraction_none_by_default(self):
state = CreditsState()
assert state.used_fraction is None
# ── depleted property ─────────────────────────────────────────────────────────
class TestDepletedProperty:
def test_depleted_equals_not_paid_access(self):
"""depleted must be exactly `not paid_access`, never `remaining==0`."""
state = CreditsState(paid_access=False, remaining_micros=0, captured_at=time.time())
assert state.depleted is True
def test_not_depleted_when_paid_access_true(self):
state = CreditsState(paid_access=True, remaining_micros=0, captured_at=time.time())
# remaining==0 but paid_access is True → NOT depleted
assert state.depleted is False
def test_depleted_independent_of_remaining(self):
"""Even with remaining > 0, if paid_access is False, depleted is True."""
state = CreditsState(paid_access=False, remaining_micros=1_000_000, captured_at=time.time())
assert state.depleted is True
# ── used_fraction edge cases ──────────────────────────────────────────────────
class TestUsedFraction:
def test_none_without_limit(self):
state = CreditsState(
denominator_kind="subscription_cap",
subscription_limit_micros=None,
captured_at=time.time(),
)
assert state.used_fraction is None
def test_none_when_limit_zero(self):
state = CreditsState(
denominator_kind="subscription_cap",
subscription_limit_micros=0,
subscription_micros=0,
captured_at=time.time(),
)
assert state.used_fraction is None
def test_clamped_at_zero(self):
"""If subscription_micros > limit (over-credited), fraction clamps to 0."""
state = CreditsState(
denominator_kind="subscription_cap",
subscription_limit_micros=10_000_000,
subscription_micros=15_000_000, # more than limit
captured_at=time.time(),
)
assert state.used_fraction == pytest.approx(0.0)
def test_clamped_at_one(self):
"""If subscription_micros is very negative (debt), fraction clamps to 1.0."""
state = CreditsState(
denominator_kind="subscription_cap",
subscription_limit_micros=10_000_000,
subscription_micros=-5_000_000, # deep debt
captured_at=time.time(),
)
assert state.used_fraction == pytest.approx(1.0)
def test_guarded_by_limit_field_not_denominator(self):
"""used_fraction depends on subscription_limit_micros being truthy, not denominator_kind."""
# limit present but denominator_kind="none" — spec says guard on LIMIT FIELD
state = CreditsState(
denominator_kind="none",
subscription_limit_micros=20_000_000,
subscription_micros=10_000_000,
captured_at=time.time(),
)
# With limit_micros set, fraction should be computable regardless of denominator_kind
assert state.used_fraction == pytest.approx(0.50)
def test_none_when_denominator_cap_but_no_limit(self):
"""denominator_kind=subscription_cap but no limit pair → None."""
state = CreditsState(
denominator_kind="subscription_cap",
subscription_limit_micros=None,
subscription_micros=5_000_000,
captured_at=time.time(),
)
assert state.used_fraction is None

5
tests/agent/test_crossloop_client_cache.py
View file

@ -10,9 +10,7 @@ so it can run without optional dependencies like firecrawl.
import asyncio
import threading
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import patch, MagicMock
from types import SimpleNamespace
import pytest
@ -32,7 +30,6 @@ def _stub_resolve_provider_client(provider, model, async_mode, **kw):
@pytest.fixture(autouse=True)
def _clean_client_cache():
"""Clear the client cache before each test."""
import importlib
# We need to patch before importing
with patch.dict("sys.modules", {}):
pass
@ -48,7 +45,7 @@ class TestCrossLoopCacheIsolation:
def test_same_loop_reuses_client(self):
"""Within a single event loop, the same client should be returned."""
from agent.auxiliary_client import _get_cached_client, _client_cache
from agent.auxiliary_client import _get_cached_client
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)

189
tests/agent/test_curator.py
View file

@ -7,7 +7,6 @@ tests run fully offline and the curator module doesn't need real credentials.
from __future__ import annotations
import importlib
import json
from datetime import datetime, timedelta, timezone
from pathlib import Path
@ -32,6 +31,11 @@ def curator_env(tmp_path, monkeypatch):
# Default: no config file → curator defaults. Tests can override.
monkeypatch.setattr(curator, "_load_config", lambda: {})
# Pin prune_builtins OFF by default so transition tests don't pick up
# built-ins unless they explicitly enable it. Both config-reading paths
# are pinned (curator reads via _load_config; skill_usage reads config
# directly). Tests opt in with _enable_prune_builtins(...).
monkeypatch.setattr(usage, "_prune_builtins_enabled", lambda: False)
return {"home": home, "curator": curator, "usage": usage}
@ -286,6 +290,170 @@ def test_bundled_skill_not_touched_by_transitions(curator_env):
assert (skills_dir / "bundled").exists() # never moved
# ---------------------------------------------------------------------------
# prune_builtins: curator may archive bundled built-ins after inactivity
# ---------------------------------------------------------------------------
def _enable_prune_builtins(curator_env, monkeypatch):
"""Flip curator.prune_builtins on for both config-reading paths."""
c = curator_env["curator"]
u = curator_env["usage"]
monkeypatch.setattr(c, "_load_config", lambda: {"prune_builtins": True})
monkeypatch.setattr(u, "_prune_builtins_enabled", lambda: True)
def _disable_prune_builtins(curator_env, monkeypatch):
"""Flip curator.prune_builtins off for both config-reading paths."""
c = curator_env["curator"]
u = curator_env["usage"]
monkeypatch.setattr(c, "_load_config", lambda: {"prune_builtins": False})
monkeypatch.setattr(u, "_prune_builtins_enabled", lambda: False)
def test_prune_builtins_default_on(curator_env):
# Shipped default is ON: with no explicit config, built-ins are eligible.
c = curator_env["curator"]
# _load_config returns {} (fixture) → default True surfaces.
assert c.get_prune_builtins() is True
def test_prune_builtins_off_excludes_bundled(curator_env, monkeypatch):
c = curator_env["curator"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "bundled")
(skills_dir / ".bundled_manifest").write_text("bundled:abc\n", encoding="utf-8")
# Explicitly off → bundled is not a candidate (the opt-out path).
_disable_prune_builtins(curator_env, monkeypatch)
assert c.get_prune_builtins() is False
counts = c.apply_automatic_transitions()
assert counts["checked"] == 0
assert (skills_dir / "bundled").exists()
def test_prune_builtins_seeds_clock_on_first_sight(curator_env, monkeypatch):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "bundled")
(skills_dir / ".bundled_manifest").write_text("bundled:abc\n", encoding="utf-8")
_enable_prune_builtins(curator_env, monkeypatch)
# First pass: built-in has no record yet → it's seeded, NOT archived,
# even though it's "old" on disk. The inactivity clock starts now.
counts = c.apply_automatic_transitions()
assert counts["checked"] == 1
assert counts["seeded"] == 1
assert counts["archived"] == 0
assert (skills_dir / "bundled").exists()
# A record now exists with created_at ~ now.
assert isinstance(u.load_usage().get("bundled"), dict)
def test_prune_builtins_archives_stale_bundled_and_suppresses(curator_env, monkeypatch):
c = curator_env["curator"]
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "bundled")
(skills_dir / ".bundled_manifest").write_text("bundled:abc\n", encoding="utf-8")
_enable_prune_builtins(curator_env, monkeypatch)
# Seed a record whose last activity is far past the archive cutoff.
super_old = (datetime.now(timezone.utc) - timedelta(days=500)).isoformat()
data = u.load_usage()
data["bundled"] = u._empty_record()
data["bundled"]["last_used_at"] = super_old
u.save_usage(data)
counts = c.apply_automatic_transitions()
assert counts["archived"] == 1
# Directory moved into .archive/, suppression recorded so update won't restore.
assert not (skills_dir / "bundled").exists()
assert (skills_dir / ".archive" / "bundled").exists()
assert "bundled" in u.read_suppressed_names()
def test_prune_builtins_restore_clears_suppression(curator_env, monkeypatch):
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "bundled")
(skills_dir / ".bundled_manifest").write_text("bundled:abc\n", encoding="utf-8")
_enable_prune_builtins(curator_env, monkeypatch)
ok, _ = u.archive_skill("bundled")
assert ok
assert "bundled" in u.read_suppressed_names()
ok, _ = u.restore_skill("bundled")
assert ok
assert (skills_dir / "bundled").exists()
assert "bundled" not in u.read_suppressed_names()
def test_protected_builtin_never_archived_even_when_stale(curator_env, monkeypatch):
"""A protected built-in (e.g. `plan`) is never archived, even when it is a
stale bundled skill under prune_builtins — it backs a load-bearing slash
command and must survive every curator pass."""
u = curator_env["usage"]
c = curator_env["curator"]
skills_dir = curator_env["home"] / "skills"
name = next(iter(u.PROTECTED_BUILTIN_SKILLS)) # the real protected name(s)
_write_skill(skills_dir, name)
(skills_dir / ".bundled_manifest").write_text(f"{name}:abc\n", encoding="utf-8")
_enable_prune_builtins(curator_env, monkeypatch)
# Force a record that is far past the archive cutoff.
super_old = (datetime.now(timezone.utc) - timedelta(days=500)).isoformat()
data = u.load_usage()
data[name] = u._empty_record()
data[name]["last_used_at"] = super_old
u.save_usage(data)
counts = c.apply_automatic_transitions()
assert counts["archived"] == 0
# Not even enumerated as a candidate → not "checked".
assert name not in u.list_agent_created_skill_names()
assert (skills_dir / name).exists()
assert name not in u.read_suppressed_names()
def test_protected_builtin_is_not_curation_eligible(curator_env, monkeypatch):
"""is_curation_eligible() returns False for protected built-ins regardless
of prune_builtins, and archive_skill() refuses them directly."""
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
name = next(iter(u.PROTECTED_BUILTIN_SKILLS))
_write_skill(skills_dir, name)
(skills_dir / ".bundled_manifest").write_text(f"{name}:abc\n", encoding="utf-8")
_enable_prune_builtins(curator_env, monkeypatch)
assert u.is_protected_builtin(name) is True
assert u.is_curation_eligible(name) is False
ok, msg = u.archive_skill(name)
assert ok is False
assert (skills_dir / name).exists()
def test_prune_builtins_never_touches_hub_skills(curator_env, monkeypatch):
u = curator_env["usage"]
skills_dir = curator_env["home"] / "skills"
_write_skill(skills_dir, "hubskill")
hub_dir = skills_dir / ".hub"
hub_dir.mkdir(parents=True, exist_ok=True)
(hub_dir / "lock.json").write_text(
'{"version": 1, "installed": {"hubskill": {"install_path": "hubskill"}}}',
encoding="utf-8",
)
_enable_prune_builtins(curator_env, monkeypatch)
# Even with prune_builtins on, hub-installed skills stay off-limits.
assert u.is_curation_eligible("hubskill") is False
ok, msg = u.archive_skill("hubskill")
assert ok is False
assert "hub-installed" in msg
assert (skills_dir / "hubskill").exists()
# ---------------------------------------------------------------------------
# run_curator_review orchestration
# ---------------------------------------------------------------------------
@ -500,8 +668,8 @@ def test_state_atomic_write_no_tmp_leftovers(curator_env):
c = curator_env["curator"]
c.save_state({"paused": True})
parent = c._state_file().parent
for p in parent.iterdir():
assert not p.name.startswith(".curator_state_"), f"tmp leftover: {p.name}"
tmp_files = [p.name for p in parent.iterdir() if p.name.endswith(".tmp")]
assert tmp_files == []
def test_state_preserves_last_report_path(curator_env):
@ -592,6 +760,21 @@ def test_curator_review_prompt_is_umbrella_first():
)
def test_curator_review_prompt_preserves_skill_package_integrity():
"""Consolidation must not flatten package skills and break linked files."""
from agent.curator import CURATOR_REVIEW_PROMPT
lower = CURATOR_REVIEW_PROMPT.lower()
assert "complete" in lower and "directory package" in lower
assert "not a new skill root" in lower
assert "do not flatten only skill.md" in lower
assert "rewrite" in lower and "new paths" in lower
assert "archive the entire original skill package unchanged" in lower
for dirname in ("references/", "templates/", "scripts/", "assets/"):
assert dirname in CURATOR_REVIEW_PROMPT
def test_curator_review_prompt_offers_support_file_actions():
"""Support-file demotion (references/templates/scripts) must be one of
the three consolidation methods, alongside merge-into-existing and

3
tests/agent/test_curator_reports.py
View file

@ -7,8 +7,7 @@ the standard log dir, not inside the user's ``skills/`` data directory.
from __future__ import annotations
import json
import os
from datetime import datetime, timezone, timedelta
from datetime import datetime, timezone
from pathlib import Path
import pytest

263
tests/agent/test_custom_providers_vision.py Normal file
View file

@ -0,0 +1,263 @@
"""Tests for custom_providers[].models[].supports_vision override (#41036).
When a named custom provider declares per-model supports_vision via the
legacy list-style custom_providers config, image_routing should honor it
and route images natively instead of falling through to models.dev or
the auxiliary vision_analyze path.
"""
from __future__ import annotations
import pytest
# ---------------------------------------------------------------------------
# _supports_vision_override — custom_providers lookup
# ---------------------------------------------------------------------------
class TestCustomProvidersVisionOverride:
"""_supports_vision_override should check custom_providers list entries."""
def test_custom_providers_supports_vision_true(self):
"""custom_providers entry with supports_vision=true → native routing."""
from agent.image_routing import _supports_vision_override
cfg = {
"custom_providers": [
{
"name": "9router-anthropic",
"models": {
"mimoanth/mimo-v2.5": {
"supports_vision": True,
}
}
}
]
}
result = _supports_vision_override(
cfg, "9router-anthropic", "mimoanth/mimo-v2.5"
)
assert result is True
def test_custom_providers_supports_vision_false(self):
"""custom_providers entry with supports_vision=False → explicit false."""
from agent.image_routing import _supports_vision_override
cfg = {
"custom_providers": [
{
"name": "my-llm",
"models": {
"some-model": {
"supports_vision": False,
}
}
}
]
}
result = _supports_vision_override(cfg, "my-llm", "some-model")
assert result is False
def test_custom_providers_custom_prefix(self):
"""Provider name at runtime may be 'custom:<name>'."""
from agent.image_routing import _supports_vision_override
cfg = {
"custom_providers": [
{
"name": "9router-anthropic",
"models": {
"mimoanth/mimo-v2.5": {
"supports_vision": True,
}
}
}
]
}
# Runtime provider is "custom:9router-anthropic"
result = _supports_vision_override(
cfg, "custom:9router-anthropic", "mimoanth/mimo-v2.5"
)
assert result is True
def test_custom_providers_no_match_returns_none(self):
"""No matching custom_providers entry → falls through (returns None)."""
from agent.image_routing import _supports_vision_override
cfg = {
"custom_providers": [
{
"name": "other-provider",
"models": {
"other-model": {
"supports_vision": True,
}
}
}
]
}
result = _supports_vision_override(
cfg, "my-provider", "my-model"
)
assert result is None
def test_custom_providers_model_not_listed(self):
"""Entry exists but model is not listed → falls through."""
from agent.image_routing import _supports_vision_override
cfg = {
"custom_providers": [
{
"name": "my-provider",
"models": {
"other-model": {
"supports_vision": True,
}
}
}
]
}
result = _supports_vision_override(
cfg, "my-provider", "unlisted-model"
)
assert result is None
def test_custom_providers_ignores_non_dict_entries(self):
"""Non-dict entries in custom_providers list are skipped."""
from agent.image_routing import _supports_vision_override
cfg = {
"custom_providers": [
"not-a-dict",
123,
None,
{
"name": "my-provider",
"models": {
"my-model": {
"supports_vision": True,
}
}
}
]
}
result = _supports_vision_override(
cfg, "my-provider", "my-model"
)
assert result is True
def test_custom_providers_empty_list(self):
"""Empty custom_providers list → no override."""
from agent.image_routing import _supports_vision_override
cfg = {"custom_providers": []}
result = _supports_vision_override(cfg, "any", "any")
assert result is None
def test_custom_providers_no_models_key(self):
"""Entry without models key → skipped gracefully."""
from agent.image_routing import _supports_vision_override
cfg = {
"custom_providers": [
{"name": "my-provider"} # no models key
]
}
result = _supports_vision_override(
cfg, "my-provider", "my-model"
)
assert result is None
def test_custom_providers_empty_name(self):
"""Entry with empty name → skipped."""
from agent.image_routing import _supports_vision_override
cfg = {
"custom_providers": [
{
"name": "",
"models": {"m": {"supports_vision": True}},
}
]
}
result = _supports_vision_override(cfg, "any", "m")
assert result is None
# ---------------------------------------------------------------------------
# decide_image_input_mode integration
# ---------------------------------------------------------------------------
class TestDecideImageInputMode:
"""End-to-end: custom_providers overrides should produce 'native' mode."""
def test_custom_providers_true_returns_native(self):
from agent.image_routing import decide_image_input_mode
cfg = {
"custom_providers": [
{
"name": "9router-anthropic",
"models": {
"mimoanth/mimo-v2.5": {
"supports_vision": True,
}
}
}
]
}
result = decide_image_input_mode(
"9router-anthropic", "mimoanth/mimo-v2.5", cfg
)
assert result == "native"
def test_custom_providers_false_returns_text(self):
from agent.image_routing import decide_image_input_mode
cfg = {
"custom_providers": [
{
"name": "my-provider",
"models": {
"my-model": {
"supports_vision": False,
}
}
}
]
}
result = decide_image_input_mode("my-provider", "my-model", cfg)
assert result == "text"
def test_top_level_supports_vision_takes_precedence(self):
"""Top-level model.supports_vision still wins over custom_providers."""
from agent.image_routing import decide_image_input_mode
cfg = {
"model": {"supports_vision": False},
"custom_providers": [
{
"name": "my-provider",
"models": {
"my-model": {
"supports_vision": True,
}
}
}
]
}
result = decide_image_input_mode("my-provider", "my-model", cfg)
assert result == "text"
def test_providers_dict_takes_precedence(self):
"""providers.<name>.models takes precedence over custom_providers."""
from agent.image_routing import decide_image_input_mode
cfg = {
"providers": {
"my-provider": {
"models": {
"my-model": {"supports_vision": False}
}
}
},
"custom_providers": [
{
"name": "my-provider",
"models": {
"my-model": {"supports_vision": True}
}
}
]
}
result = decide_image_input_mode("my-provider", "my-model", cfg)
assert result == "text"

3
tests/agent/test_display.py
View file

@ -1,9 +1,8 @@
"""Tests for agent/display.py — build_tool_preview() and inline diff previews."""
import os
import json
import pytest
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
from agent.display import (
build_tool_preview,

242
tests/agent/test_display_todo_progress.py Normal file
View file

@ -0,0 +1,242 @@
"""Tests for get_cute_tool_message todo progress display.
Verifies the completion status rendering (done/total ✓) on all three
todo tool call paths: read, create (merge=False), update (merge=True).
"""
import json
from agent.display import get_cute_tool_message
def _todo_result(total: int, completed: int) -> str:
"""Build a fake todo_tool return value."""
return json.dumps({
"todos": [],
"summary": {
"total": total,
"pending": total - completed,
"in_progress": 0,
"completed": completed,
"cancelled": 0,
},
})
class TestTodoRead:
"""get_cute_tool_message(…, result=…) when todos_arg is None (read path)."""
def test_read_no_result(self):
msg = get_cute_tool_message("todo", {}, 0.5)
assert "reading tasks" in msg
assert "0.5s" in msg
def test_read_with_progress(self):
msg = get_cute_tool_message("todo", {}, 0.5,
result=_todo_result(4, 2))
assert "2/4" in msg
assert "task(s)" in msg
def test_read_all_done(self):
msg = get_cute_tool_message("todo", {}, 0.5,
result=_todo_result(4, 4))
assert "4/4" in msg
assert "task(s)" in msg
def test_read_zero_total(self):
"""Edge case: empty todo list returns summary with total=0."""
msg = get_cute_tool_message("todo", {}, 0.5,
result=_todo_result(0, 0))
assert "reading tasks" in msg
def test_read_invalid_result_fallback(self):
"""Garbage result should not crash; fall back to reading tasks."""
msg = get_cute_tool_message("todo", {}, 0.5, result="not json")
assert "reading tasks" in msg
def test_read_result_missing_summary(self):
msg = get_cute_tool_message("todo", {}, 0.5,
result='{"todos": []}')
assert "reading tasks" in msg
class TestTodoCreate:
"""get_cute_tool_message when merge=False (new plan creation)."""
def test_create_default(self):
"""Brand-new plan: all pending, no result — plain count."""
msg = get_cute_tool_message("todo",
{"todos": [
{"id": "a", "content": "x", "status": "pending"},
]}, 0.3)
assert "1 task(s)" in msg
assert "0.3s" in msg
assert "/" not in msg # no progress fraction
def test_create_multiple(self):
msg = get_cute_tool_message("todo",
{"todos": [
{"id": "a", "content": "x", "status": "pending"},
{"id": "b", "content": "y", "status": "pending"},
{"id": "c", "content": "z", "status": "pending"},
]}, 0.2)
assert "3 task(s)" in msg
def test_create_with_result_shows_progress_when_done(self):
"""Even on create, if result has completed tasks show it."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "content": "x", "status": "completed"}]},
0.4,
result=_todo_result(1, 1))
assert "1/1" in msg
assert "task(s)" in msg
def test_create_with_result_zero_done(self):
"""New plan with 0 done — plain count, no progress fraction."""
msg = get_cute_tool_message("todo",
{"todos": [
{"id": "a", "content": "x", "status": "pending"},
{"id": "b", "content": "y", "status": "pending"},
]},
0.3,
result=_todo_result(2, 0))
assert "2 task(s)" in msg
assert "/" not in msg
class TestTodoUpdate:
"""get_cute_tool_message when merge=True (incremental update)."""
def test_update_no_result(self):
"""No result available — plain update N task(s)."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True}, 0.5)
assert "update 1 task(s)" in msg
def test_update_partial_progress(self):
"""1/4 tasks completed — show fraction with checkmark."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True},
0.5,
result=_todo_result(4, 1))
assert "update" in msg
assert "1/4" in msg
assert "✓" in msg
def test_update_halfway(self):
"""2/4 — midpoint progress."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "b", "status": "in_progress"}],
"merge": True},
0.7,
result=_todo_result(4, 2))
assert "2/4" in msg
assert "✓" in msg
def test_update_all_completed(self):
"""4/4 — full checkmark."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "d", "status": "completed"}],
"merge": True},
0.2,
result=_todo_result(4, 4))
assert "4/4" in msg
assert "✓" in msg
def test_update_zero_done(self):
"""No completed tasks yet — plain update N task(s)."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "pending"}],
"merge": True},
0.3,
result=_todo_result(3, 0))
assert "update 1 task(s)" in msg
assert "✓" not in msg
assert "/" not in msg # no progress fraction when done=0
def test_update_invalid_result_fallback(self):
"""Bad JSON result — fall back to plain update N task(s)."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True},
0.6,
result="{broken")
assert "update 1 task(s)" in msg
assert "✓" not in msg
def test_update_result_missing_summary(self):
"""Result no summary key — fall back to plain update."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True},
0.4,
result='{"todos": []}')
assert "update 1 task(s)" in msg
assert "✓" not in msg
def test_update_total_not_in_summary(self):
"""Result summary missing total key."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "status": "completed"}],
"merge": True},
0.3,
result=json.dumps({"summary": {"completed": 2}}))
assert "update 1 task(s)" in msg
assert "✓" not in msg
def test_update_multiple_tasks_in_line(self):
"""Update line with several tasks in the update request."""
msg = get_cute_tool_message("todo",
{"todos": [
{"id": "a", "status": "completed"},
{"id": "b", "status": "in_progress"},
], "merge": True},
0.5,
result=_todo_result(5, 3))
assert "update" in msg
assert "3/5" in msg
assert "✓" in msg
class TestTodoEdgeCases:
"""Boundary cases that should not crash."""
def test_merge_default_value(self):
"""merge defaults to False in function signature, should be False when absent."""
msg = get_cute_tool_message("todo",
{"todos": [{"id": "a", "content": "x", "status": "pending"}]},
1.0)
assert "1 task(s)" in msg
def test_duration_formatting(self):
"""Duration formatting works correctly."""
msg = get_cute_tool_message("todo", {}, 0.123)
assert "0.1s" in msg
msg = get_cute_tool_message("todo", {}, 1.0)
assert "1.0s" in msg
msg = get_cute_tool_message("todo", {}, 123.456)
assert "123.5s" in msg
def test_large_task_count(self):
"""Many tasks should not break formatting."""
many = [{"id": str(i), "content": "x", "status": "pending"} for i in range(50)]
msg = get_cute_tool_message("todo", {"todos": many}, 0.5)
assert "50 task(s)" in msg
def test_read_with_no_args_and_no_result(self):
"""Completely empty call."""
msg = get_cute_tool_message("todo", {}, 0.0)
assert "reading tasks" in msg
class TestTodoSkinIntegration:
"""Verify the skin prefix is applied to todo messages too.
This uses the same pattern as test_skin_engine test_tool_message_uses_skin_prefix.
"""
def test_default_skin_prefix(self):
msg = get_cute_tool_message("todo", {}, 0.5)
assert msg.startswith("┊")

184
tests/agent/test_display_tool_failure.py Normal file
View file

@ -0,0 +1,184 @@
"""Tests for _detect_tool_failure + _trim_error + get_cute_tool_message
inline failure suffix rendering.
Covers the user-visible promise: when a tool fails, the CLI shows a short,
specific reason in square brackets at the end of the completion line —
not a generic "[error]".
"""
import json
from agent.display import (
_detect_tool_failure,
_trim_error,
_ERROR_SUFFIX_MAX_LEN,
get_cute_tool_message,
)
class TestTrimError:
"""The helper that shrinks an error message for inline display."""
def test_short_message_unchanged(self):
assert _trim_error("nope") == "nope"
def test_whitespace_stripped(self):
assert _trim_error(" bad input ") == "bad input"
def test_long_message_truncated_to_cap(self):
msg = "x" * 200
trimmed = _trim_error(msg)
assert len(trimmed) <= _ERROR_SUFFIX_MAX_LEN
assert trimmed.endswith("...")
def test_file_not_found_path_collapsed_to_filename(self):
long_path = "File not found: /home/teknium/.hermes/hermes-agent/very/deep/path/foo.py"
assert _trim_error(long_path) == "File not found: foo.py"
def test_file_not_found_already_short_unchanged(self):
assert _trim_error("File not found: foo.py") == "File not found: foo.py"
def test_file_not_found_relative_path_unchanged(self):
# Without a slash there's no path to trim.
assert _trim_error("File not found: foo.py") == "File not found: foo.py"
class TestDetectToolFailureTerminal:
"""terminal: non-zero exit_code is the canonical failure signal."""
def test_success_returns_no_suffix(self):
result = json.dumps({"output": "ok\n", "exit_code": 0})
assert _detect_tool_failure("terminal", result) == (False, "")
def test_nonzero_exit_with_no_error_shows_exit_code(self):
result = json.dumps({"output": "", "exit_code": 1})
is_failure, suffix = _detect_tool_failure("terminal", result)
assert is_failure is True
assert suffix == " [exit 1]"
def test_nonzero_exit_with_error_shows_message(self):
result = json.dumps({
"output": "",
"exit_code": 127,
"error": "ls: cannot access 'foo': No such file or directory",
})
is_failure, suffix = _detect_tool_failure("terminal", result)
assert is_failure is True
assert "cannot access" in suffix
# Trimmed to the cap, in brackets
assert suffix.startswith(" [")
assert suffix.endswith("]")
def test_malformed_json_returns_no_suffix(self):
# Terminal is special: only exit_code matters. Malformed JSON should
# not crash and should not be flagged as failure.
assert _detect_tool_failure("terminal", "not json") == (False, "")
def test_none_result_returns_no_suffix(self):
assert _detect_tool_failure("terminal", None) == (False, "")
class TestDetectToolFailureMemory:
"""memory: 'full' is distinct from real errors."""
def test_memory_full_returns_full_suffix(self):
result = json.dumps({"success": False, "error": "would exceed the limit"})
assert _detect_tool_failure("memory", result) == (True, " [full]")
def test_memory_other_error_returns_specific_message(self):
# An error that's NOT a "full" overflow falls through to the
# structured-error path and surfaces the actual message.
result = json.dumps({"success": False, "error": "invalid action: zap"})
is_failure, suffix = _detect_tool_failure("memory", result)
assert is_failure is True
assert "invalid action" in suffix
class TestDetectToolFailureStructured:
"""Generic path: any tool that returns {"error": ...} JSON."""
def test_read_file_error_surfaced(self):
result = json.dumps({
"path": "/nope/missing.py",
"success": False,
"error": "File not found: /nope/missing.py",
})
is_failure, suffix = _detect_tool_failure("read_file", result)
assert is_failure is True
# _trim_error reduces the path to the basename.
assert suffix == " [File not found: missing.py]"
def test_error_without_success_key_still_flagged(self):
# Some tools return {"error": "..."} with no explicit success flag.
result = json.dumps({"error": "remote unavailable"})
is_failure, suffix = _detect_tool_failure("web_search", result)
assert is_failure is True
assert suffix == " [remote unavailable]"
def test_message_field_only_with_success_false_flagged(self):
# When success is False and only 'message' is set, surface it.
result = json.dumps({"success": False, "message": "rate limited"})
is_failure, suffix = _detect_tool_failure("web_search", result)
assert is_failure is True
assert "rate limited" in suffix
def test_successful_result_not_flagged(self):
result = json.dumps({"success": True, "data": "hello"})
assert _detect_tool_failure("web_search", result) == (False, "")
def test_dict_without_error_or_success_uses_generic_heuristic(self):
# Plain successful dict — should pass through the generic
# heuristic which only fires on the string "Error" / '"error"' / etc.
result = json.dumps({"data": "hello"})
is_failure, _ = _detect_tool_failure("web_search", result)
assert is_failure is False
class TestGetCuteToolMessageFailureSuffix:
"""End-to-end: failure suffix is appended by get_cute_tool_message."""
def test_read_file_failure_suffix_appended(self):
fail = json.dumps({
"path": "/etc/missing",
"success": False,
"error": "File not found: /etc/missing",
})
line = get_cute_tool_message("read_file", {"path": "/etc/missing"}, 0.1, result=fail)
assert "[File not found: missing]" in line
def test_terminal_exit_only_suffix(self):
fail = json.dumps({"output": "", "exit_code": 2})
line = get_cute_tool_message("terminal", {"command": "false"}, 0.1, result=fail)
assert "[exit 2]" in line
def test_terminal_with_stderr_uses_message(self):
fail = json.dumps({
"output": "",
"exit_code": 127,
"error": "command not found: notathing",
})
line = get_cute_tool_message("terminal", {"command": "notathing"}, 0.1, result=fail)
assert "command not found" in line
# No '[exit 127]' tag when we have a specific message
assert "exit 127" not in line
def test_memory_full_suffix(self):
fail = json.dumps({"success": False, "error": "would exceed the limit"})
line = get_cute_tool_message(
"memory",
{"action": "add", "target": "memory", "content": "x"},
0.05,
result=fail,
)
assert "[full]" in line
def test_success_has_no_suffix(self):
ok = json.dumps({"success": True, "data": "hi"})
line = get_cute_tool_message("web_search", {"query": "hi"}, 0.2, result=ok)
assert "[" not in line.split("0.2s", 1)[1]
def test_no_result_has_no_suffix(self):
# No result passed at all — display function should not invent a
# failure suffix.
line = get_cute_tool_message("terminal", {"command": "ls"}, 0.2)
assert "[" not in line.split("0.2s", 1)[1]

362
tests/agent/test_error_classifier.py
View file

@ -56,7 +56,10 @@ class TestFailoverReason:
"overloaded", "server_error", "timeout",
"context_overflow", "payload_too_large", "image_too_large",
"model_not_found", "format_error",
"invalid_encrypted_content",
"multimodal_tool_content_unsupported",
"provider_policy_blocked",
"content_policy_blocked",
"thinking_signature", "long_context_tier",
"oauth_long_context_beta_forbidden",
"llama_cpp_grammar_pattern",
@ -143,6 +146,19 @@ class TestExtractErrorCode:
body = {"code": "model_not_found"}
assert _extract_error_code(body) == "model_not_found"
def test_from_wrapped_json_message(self):
body = {
"error": {
"message": (
'{"error":{"message":"The encrypted content for item rs_001 could not be verified. '
'Reason: Encrypted content could not be decrypted or parsed.",'
'"type":"invalid_request_error","param":"","code":"invalid_encrypted_content"}}'
),
"type": "400",
}
}
assert _extract_error_code(body) == "invalid_encrypted_content"
def test_empty_when_no_code(self):
assert _extract_error_code({}) == ""
assert _extract_error_code({"error": {"message": "oops"}}) == ""
@ -239,12 +255,51 @@ class TestClassifyApiError:
assert result.reason == FailoverReason.billing
assert result.retryable is False
def test_402_out_of_funds_billing(self):
e = MockAPIError(
"Payment Required",
status_code=402,
body={
"status": 402,
"message": (
"Your API key has run out of funds. Please go visit the "
"portal to sort that out: https://portal.nousresearch.com"
),
},
)
result = classify_api_error(e)
assert result.reason == FailoverReason.billing
assert result.retryable is False
def test_402_transient_usage_limit(self):
e = MockAPIError("usage limit exceeded, try again later", status_code=402)
result = classify_api_error(e)
assert result.reason == FailoverReason.rate_limit
assert result.retryable is True
def test_403_plan_entitlement_billing(self):
e = MockAPIError("This plan does not include the requested model", status_code=403)
result = classify_api_error(e)
assert result.reason == FailoverReason.billing
assert result.retryable is False
def test_404_free_tier_model_block_is_billing(self):
e = MockAPIError(
"Not Found",
status_code=404,
body={
"status": 404,
"message": (
"Model 'gpt-5' is not available on the Free Tier. "
"Upgrade at https://portal.nousresearch.com or pick a free model."
),
},
)
result = classify_api_error(e, provider="nous", model="gpt-5")
assert result.reason == FailoverReason.billing
assert result.retryable is False
assert result.should_fallback is True
# ── Rate limit ──
def test_429_rate_limit(self):
@ -292,6 +347,64 @@ class TestClassifyApiError:
result = classify_api_error(e)
assert result.reason == FailoverReason.overloaded
# ── 5xx that are actually request-validation errors ──
# Some OpenAI-compatible gateways (e.g. codex.nekos.me) return
# request-validation failures with a 5xx status. These are
# deterministic, so they must NOT be retried — otherwise the retry
# loop hammers the identical bad request into a flood.
def test_502_with_unknown_parameter_is_non_retryable(self):
e = MockAPIError(
"Unknown parameter: 'input[617]._empty_recovery_synthetic'",
status_code=502,
body={
"error": {
"type": "invalid_request_error",
"message": (
"[ObjectParam] [input[617]._empty_recovery_synthetic] "
"[unknown_parameter] Unknown parameter: "
"'input[617]._empty_recovery_synthetic'."
),
}
},
)
result = classify_api_error(e)
assert result.reason == FailoverReason.format_error
assert result.retryable is False
assert result.should_fallback is True
def test_502_with_unsupported_parameter_is_non_retryable(self):
e = MockAPIError(
"Unsupported parameter: logprobs",
status_code=502,
body={
"error": {
"type": "invalid_request_error",
"message": "Unsupported parameter: logprobs",
}
},
)
result = classify_api_error(e)
assert result.reason == FailoverReason.format_error
assert result.retryable is False
def test_500_with_invalid_request_error_type_is_non_retryable(self):
e = MockAPIError(
"bad request",
status_code=500,
body={"error": {"type": "invalid_request_error", "message": "bad request"}},
)
result = classify_api_error(e)
assert result.reason == FailoverReason.format_error
assert result.retryable is False
def test_502_plain_bad_gateway_still_retryable(self):
"""A genuine 502 with no request-validation signal stays retryable."""
e = MockAPIError("Bad Gateway", status_code=502)
result = classify_api_error(e)
assert result.reason == FailoverReason.server_error
assert result.retryable is True
# ── Model not found ──
def test_404_model_not_found(self):
@ -354,6 +467,78 @@ class TestClassifyApiError:
result = classify_api_error(e)
assert result.reason == FailoverReason.provider_policy_blocked
# ── Provider content-policy block (per-prompt safety filter) ──
#
# Distinct from ``provider_policy_blocked`` above — these are upstream
# model-provider safety refusals for THIS prompt, not OpenRouter
# account-level data policy. Recovery is fallback model, not config fix.
# See issue #18028 — OpenAI Codex was burning 3 retries on identical
# refusals before users saw "API failed after 3 retries" on Telegram.
def test_message_only_cyber_content_policy_blocked(self):
# OpenAI Codex returns this without an HTTP status. Retrying the
# same prompt three times only repeats the same policy decision, so
# the classifier must jump straight to fallback / abort instead of
# leaving it in the retryable ``unknown`` bucket.
e = Exception(
"This content was flagged for possible cybersecurity risk. If this "
"seems wrong, try rephrasing your request. To get authorized for "
"security work, join the Trusted Access for Cyber program."
)
result = classify_api_error(e, provider="openai-codex", model="gpt-5.5")
assert result.reason == FailoverReason.content_policy_blocked
assert result.retryable is False
assert result.should_fallback is True
assert result.should_compress is False
def test_400_cyber_content_policy_blocked(self):
# When the SDK does attach a status (e.g. 400), the safety pattern
# must still beat the format_error fallthrough.
e = MockAPIError(
"This content was flagged for possible cybersecurity risk",
status_code=400,
)
result = classify_api_error(e, provider="openai-codex", model="gpt-5.5")
assert result.reason == FailoverReason.content_policy_blocked
assert result.retryable is False
assert result.should_fallback is True
def test_openai_usage_policy_violation_content_policy_blocked(self):
# OpenAI moderation refusal wording from chat completions / responses.
e = MockAPIError(
"Your request was flagged by the moderation system as potentially "
"violating OpenAI's usage policies.",
status_code=400,
)
result = classify_api_error(e, provider="openai", model="gpt-4o")
assert result.reason == FailoverReason.content_policy_blocked
assert result.retryable is False
assert result.should_fallback is True
def test_anthropic_safety_system_content_policy_blocked(self):
# Anthropic safety refusal — distinct phrasing from OpenAI.
e = Exception(
"Your prompt was flagged by our safety system. Please rephrase "
"and try again."
)
result = classify_api_error(e, provider="anthropic", model="claude-3-5-sonnet")
assert result.reason == FailoverReason.content_policy_blocked
assert result.retryable is False
assert result.should_fallback is True
def test_azure_content_filter_content_policy_blocked(self):
# Azure OpenAI returns ``content_filter`` finish reason / error code
# and ``ResponsibleAIPolicyViolation`` in error bodies — both narrow
# tokens, not the generic English phrase.
e = MockAPIError(
"The response was filtered: ResponsibleAIPolicyViolation "
"(finish_reason=content_filter).",
status_code=400,
)
result = classify_api_error(e, provider="azure", model="gpt-4o")
assert result.reason == FailoverReason.content_policy_blocked
assert result.retryable is False
def test_404_model_not_found_still_works(self):
# Regression guard: the new policy-block check must not swallow
# genuine model_not_found 404s.
@ -476,6 +661,51 @@ class TestClassifyApiError:
# Without "thinking" in the message, it shouldn't be thinking_signature
assert result.reason != FailoverReason.thinking_signature
def test_invalid_encrypted_content_classified_as_retryable_replay_failure(self):
body = {
"error": {
"message": (
'{"error":{"message":"The encrypted content for item rs_001 could not be verified. '
'Reason: Encrypted content could not be decrypted or parsed.",'
'"type":"invalid_request_error","param":"","code":"invalid_encrypted_content"}}'
),
"type": "400",
}
}
e = MockAPIError(
"Error code: 400 - invalid_encrypted_content",
status_code=400,
body=body,
)
result = classify_api_error(e, provider="custom", model="gpt-5.4")
assert result.reason == FailoverReason.invalid_encrypted_content
assert result.retryable is True
assert result.should_fallback is False
def test_invalid_encrypted_content_broad_message_match_does_not_catch_generic_parse_error(self):
message = "Encrypted content could not be decrypted or parsed."
e = MockAPIError(
message,
status_code=400,
body={"error": {"message": message}},
)
result = classify_api_error(e, provider="custom", model="gpt-5.4")
assert result.reason == FailoverReason.format_error
assert result.retryable is False
assert result.should_fallback is True
@pytest.mark.parametrize("error_code", ["Invalid_Encrypted_Content", "INVALID_ENCRYPTED_CONTENT"])
def test_invalid_encrypted_content_code_is_case_insensitive_for_400(self, error_code):
e = MockAPIError(
"Error code: 400 - bad request",
status_code=400,
body={"error": {"code": error_code, "message": "Bad request"}},
)
result = classify_api_error(e, provider="custom", model="gpt-5.4")
assert result.reason == FailoverReason.invalid_encrypted_content
assert result.retryable is True
assert result.should_fallback is False
# ── Provider-specific: llama.cpp grammar-parse ──
def test_llama_cpp_grammar_parse_error(self):
@ -635,6 +865,19 @@ class TestClassifyApiError:
result = classify_api_error(e)
assert result.reason == FailoverReason.context_overflow
def test_error_code_model_not_supported_on_free_tier_is_billing(self):
e = MockAPIError(
"Model unavailable",
body={
"error": {
"code": "model_not_supported_on_free_tier",
"message": "Model 'gpt-5' is not available on the Free Tier.",
}
},
)
result = classify_api_error(e, provider="nous", model="gpt-5")
assert result.reason == FailoverReason.billing
# ── Message-only patterns (no status code) ──
def test_message_billing_pattern(self):
@ -642,6 +885,11 @@ class TestClassifyApiError:
result = classify_api_error(e)
assert result.reason == FailoverReason.billing
def test_message_free_tier_model_block_is_billing(self):
e = Exception("Model 'gpt-5' is not available on the Free Tier.")
result = classify_api_error(e, provider="nous", model="gpt-5")
assert result.reason == FailoverReason.billing
def test_message_rate_limit_pattern(self):
e = Exception("rate limit reached for this model")
result = classify_api_error(e)
@ -716,6 +964,57 @@ class TestClassifyApiError:
assert result.reason == FailoverReason.format_error
assert result.retryable is False
def test_400_unsupported_max_tokens_param_not_context_overflow(self):
"""A GPT-5 model rejecting max_tokens must NOT be misclassified as
context overflow. The OpenAI error string contains the literal
'max_tokens' (a _CONTEXT_OVERFLOW_PATTERNS entry), so without the
request-validation guard it was routed into the compression loop,
re-sent with the same bad param, and ended in "Cannot compress
further". Regression for gpt-5-context-overflow-misclassification."""
msg = ("Unsupported parameter: 'max_tokens' is not supported with this "
"model. Use 'max_completion_tokens' instead.")
e = MockAPIError(
msg,
status_code=400,
body={"error": {"message": msg, "type": "invalid_request_error",
"code": "unsupported_parameter"}},
)
# Tiny context against a huge window — definitely not a real overflow.
result = classify_api_error(e, model="gpt-5.4",
approx_tokens=6962, context_length=1050000)
assert result.reason == FailoverReason.format_error
assert result.retryable is False
assert result.should_compress is False
def test_400_unknown_parameter_not_context_overflow(self):
"""'Unknown parameter' 400s are deterministic request-validation
failures, not overflows."""
e = MockAPIError(
"Unknown parameter: 'foo'.",
status_code=400,
body={"error": {"message": "Unknown parameter: 'foo'.",
"code": "unknown_parameter"}},
)
result = classify_api_error(e, approx_tokens=1000)
assert result.reason == FailoverReason.format_error
assert result.should_compress is False
def test_400_real_overflow_with_invalid_request_error_code_still_compresses(self):
"""Guard the guard: OpenAI stamps genuine context-overflow 400s with
the generic 'invalid_request_error' code. The request-validation guard
must NOT key off that code, or real overflows stop compressing."""
msg = ("This model's maximum context length is 128000 tokens, however "
"you requested 150000 tokens.")
e = MockAPIError(
msg,
status_code=400,
body={"error": {"message": msg, "type": "invalid_request_error"}},
)
result = classify_api_error(e, model="gpt-5.4",
approx_tokens=150000, context_length=128000)
assert result.reason == FailoverReason.context_overflow
assert result.should_compress is True
def test_422_format_error(self):
e = MockAPIError("Unprocessable Entity", status_code=422)
result = classify_api_error(e)
@ -1256,3 +1555,66 @@ class TestRateLimitErrorWithoutStatusCode:
e.status_code = None
result = classify_api_error(e, provider="copilot", model="gpt-4o")
assert result.reason != FailoverReason.rate_limit
# ── Test: multimodal_tool_content_unsupported pattern ───────────────────
class TestMultimodalToolContentUnsupported:
"""Issue #27344 — providers that reject list-type tool message content
should be classified as ``multimodal_tool_content_unsupported`` so the
retry loop can downgrade screenshots to text and try again.
"""
def test_xiaomi_mimo_text_is_not_set_pattern(self):
"""The actual Xiaomi MiMo 400 wording from the bug report."""
e = MockAPIError(
"Error code: 400 - {'error': {'code': '400', 'message': 'Param Incorrect', 'param': 'text is not set', 'type': ''}}",
status_code=400,
)
result = classify_api_error(e, provider="xiaomi", model="mimo-v2.5")
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
assert result.retryable is True
def test_generic_tool_message_must_be_string(self):
e = MockAPIError(
"tool message content must be a string",
status_code=400,
)
result = classify_api_error(e, provider="custom", model="some-model")
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
def test_expected_string_got_list(self):
e = MockAPIError(
"Schema validation failed: expected string, got list",
status_code=400,
)
result = classify_api_error(e, provider="custom", model="some-model")
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
def test_multimodal_tool_content_takes_priority_over_context_overflow(self):
"""Some providers return a 400 whose message contains BOTH
'text is not set' and a length-shaped phrase; the tool-content
recovery is cheaper than compression so it must win the priority.
"""
e = MockAPIError(
"text is not set; context length exceeded",
status_code=400,
)
result = classify_api_error(e, provider="xiaomi", model="mimo-v2.5")
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
def test_no_status_code_path_also_classifies(self):
"""When the error reaches us without a status code (transport
layer ate it) the message-only classifier branch must also
recognise the pattern.
"""
e = MockTransportError("tool_call.content must be string")
result = classify_api_error(e, provider="alibaba", model="qwen3.5-plus")
assert result.reason == FailoverReason.multimodal_tool_content_unsupported
def test_unrelated_400_is_not_misclassified(self):
"""Make sure the patterns don't false-positive on normal 400s."""
e = MockAPIError("bad request: missing field 'model'", status_code=400)
result = classify_api_error(e, provider="openrouter", model="anthropic/claude-sonnet-4")
assert result.reason != FailoverReason.multimodal_tool_content_unsupported

1
tests/agent/test_external_skills.py
View file

@ -2,7 +2,6 @@
import json
import os
from pathlib import Path
from unittest.mock import patch
import pytest

1
tests/agent/test_external_skills_dirs_cache.py
View file

@ -11,7 +11,6 @@ cache invalidates when config.yaml's mtime changes.
from __future__ import annotations
import os
import time
from pathlib import Path
from unittest.mock import patch

148
tests/agent/test_file_safety.py Normal file
View file

@ -0,0 +1,148 @@
"""Tests for agent/file_safety.py read guards — env file blocking.
Run with: python -m pytest tests/agent/test_file_safety.py -v
"""
import os
from unittest.mock import patch
import pytest
from agent.file_safety import (
_BLOCKED_PROJECT_ENV_BASENAMES,
get_read_block_error,
)
# ---------------------------------------------------------------------------
# Project-local .env file blocking (issue #20734)
# ---------------------------------------------------------------------------
class TestEnvFileReadBlocking:
"""Secret-bearing .env files must be blocked by get_read_block_error."""
@pytest.mark.parametrize("basename", [
".env",
".env.local",
".env.development",
".env.production",
".env.test",
".env.staging",
".envrc",
])
def test_blocked_env_basenames(self, basename):
"""All secret-bearing .env basenames are blocked regardless of directory."""
path = f"/tmp/project/{basename}"
error = get_read_block_error(path)
assert error is not None, f"{basename} should be blocked"
assert "Access denied" in error
assert "secret-bearing" in error.lower() or "environment file" in error.lower()
def test_blocked_env_in_subdirectory(self):
"""Nested .env files are also blocked."""
error = get_read_block_error("/home/user/app/services/api/.env.production")
assert error is not None
def test_blocked_env_absolute_path(self):
"""Absolute paths to .env files are blocked."""
error = get_read_block_error("/opt/myapp/.env")
assert error is not None
def test_allowed_env_example(self):
""""The .env.example file is explicitly allowed — it's documentation, not a secret."""
error = get_read_block_error("/tmp/project/.env.example")
assert error is None
def test_allowed_env_sample(self):
"""Other .env variants like .env.sample are allowed."""
error = get_read_block_error("/tmp/project/.env.sample")
assert error is None
def test_allowed_non_env_files(self):
"""Regular files are not affected by the env guard."""
for path in ["/tmp/project/config.yaml", "/tmp/project/main.py",
"/tmp/project/README.md", "/tmp/project/.gitignore"]:
error = get_read_block_error(path)
assert error is None, f"{path} should be allowed"
def test_allowed_hermes_env(self):
"""Hermes' own .env inside HERMES_HOME is NOT blocked by this rule
(it's handled by other mechanisms). Only project-local .env is blocked."""
# Note: hermes internal .env is in ~/.hermes/.env which is NOT a project-local
# path, but the basename check applies to ANY .env. This is intentional —
# even ~/.hermes/.env should not be readable via read_file.
error = get_read_block_error(os.path.expanduser("~/.hermes/.env"))
assert error is not None
def test_blocked_set_is_lowercase(self):
"""All entries in the blocked set are lowercase for case-insensitive matching."""
for name in _BLOCKED_PROJECT_ENV_BASENAMES:
assert name == name.lower(), f"{name} should be lowercase"
# ---------------------------------------------------------------------------
# Existing cache-file blocking (regression — must still work)
# ---------------------------------------------------------------------------
class TestCacheFileReadBlocking:
"""Internal Hermes cache files must remain blocked."""
def test_hub_index_cache_blocked(self, tmp_path):
"""Hub index-cache reads are blocked."""
hermes_home = tmp_path / ".hermes"
cache = hermes_home / "skills" / ".hub" / "index-cache" / "data.json"
cache.parent.mkdir(parents=True)
cache.write_text("{}")
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
error = get_read_block_error(str(cache))
assert error is not None
assert "internal Hermes cache" in error
def test_hub_directory_blocked(self, tmp_path):
"""Hub directory reads are blocked."""
hermes_home = tmp_path / ".hermes"
hub = hermes_home / "skills" / ".hub" / "metadata.json"
hub.parent.mkdir(parents=True)
hub.write_text("{}")
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
error = get_read_block_error(str(hub))
assert error is not None
# ---------------------------------------------------------------------------
# Combined: env guard + cache guard don't interfere
# ---------------------------------------------------------------------------
class TestCombinedGuards:
"""Both guards should work independently without interference."""
def test_env_guard_works_regardless_of_hermes_home(self, tmp_path):
"""The env basename guard does not depend on HERMES_HOME resolution."""
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
# Regular project .env should still be blocked
error = get_read_block_error("/workspace/.env")
assert error is not None
# .env.example should still be allowed
error = get_read_block_error("/workspace/.env.example")
assert error is None
def test_cache_guard_still_works_with_env_guard(self, tmp_path):
"""Cache file blocking still works when env guard is active."""
hermes_home = tmp_path / ".hermes"
cache = hermes_home / "skills" / ".hub" / "index-cache" / "x"
cache.parent.mkdir(parents=True)
cache.write_text("")
with patch("agent.file_safety._hermes_home_path", return_value=hermes_home):
error = get_read_block_error(str(cache))
assert error is not None
assert "internal Hermes cache" in error

106
tests/agent/test_file_safety_container_mirror.py Normal file
View file

@ -0,0 +1,106 @@
"""Tests for the container-context sandbox-mirror guard (#32049 follow-up).
Brian's shape-based guard (#32213) catches paths that carry the full
``…/sandboxes/<backend>/<task>/home/.hermes/…`` prefix. This covers the
complementary inner-container case: when file tools execute inside Docker,
the bind-mount strips that prefix and the guard sees plain ``/root/.hermes/…``.
The root:root ownership on the divergent SOUL.md in #32049 confirms this
is the primary failure mode.
"""
from __future__ import annotations
import pytest
class TestClassifyContainerMirrorTarget:
def test_returns_none_without_context(self):
"""No Docker context — /root/.hermes/… must not be flagged."""
from agent.file_safety import classify_container_mirror_target
assert classify_container_mirror_target("/root/.hermes/profiles/group1/SOUL.md") is None
def test_catches_soul_md_with_context(self):
"""Primary failure mode from #32049: agent writes SOUL.md via container path."""
from agent.file_safety import classify_container_mirror_target
result = classify_container_mirror_target(
"/root/.hermes/profiles/group1/SOUL.md",
mirror_prefix="/root/.hermes",
)
assert result is not None
assert result["mirror_root"].replace("\\", "/").endswith("root/.hermes")
assert result["inner_path"] == "profiles/group1/SOUL.md"
@pytest.mark.parametrize("inner", [
"SOUL.md",
"memories/MEMORY.md",
])
def test_catches_authoritative_profile_files(self, inner):
from agent.file_safety import classify_container_mirror_target
result = classify_container_mirror_target(
f"/root/.hermes/{inner}",
mirror_prefix="/root/.hermes",
)
assert result is not None
assert result["inner_path"] == inner
def test_non_hermes_path_not_flagged(self):
"""/root/workspace/… is not .hermes state and must not be blocked."""
from agent.file_safety import classify_container_mirror_target
assert (
classify_container_mirror_target(
"/root/workspace/main.py",
mirror_prefix="/root/.hermes",
)
is None
)
class TestGetContainerMirrorWarning:
def test_warning_names_inner_path_and_bypass(self):
from agent.file_safety import get_container_mirror_warning
warn = get_container_mirror_warning(
"/root/.hermes/profiles/group1/SOUL.md",
mirror_prefix="/root/.hermes",
)
assert warn is not None
assert "profiles/group1/SOUL.md" in warn
assert "cross_profile=True" in warn
class TestOrthogonality:
"""Container-context guard catches what the shape-based guard (#32213) misses."""
def test_inner_container_path_caught_by_context_guard(self):
"""No sandboxes/ segment — shape guard passes, context guard blocks."""
from agent.file_safety import classify_container_mirror_target
path = "/root/.hermes/profiles/group1/SOUL.md"
assert classify_container_mirror_target(path) is None # no context
assert classify_container_mirror_target(path, mirror_prefix="/root/.hermes") is not None
class TestFileToolIntegration:
"""file_tools must catch the mirror path before creating DockerEnvironment."""
def test_guard_uses_current_docker_config_before_env_exists(self, monkeypatch):
import tools.file_tools as file_tools
monkeypatch.setattr(
file_tools,
"_get_container_mirror_prefix_for_task",
lambda task_id: "/root/.hermes",
)
warning = file_tools._check_cross_profile_path(
"/root/.hermes/profiles/group1/SOUL.md",
task_id="new-task",
)
assert warning is not None
assert "Sandbox-mirror write blocked" in warning
assert "profiles/group1/SOUL.md" in warning

339
tests/agent/test_file_safety_credentials.py Normal file
View file

@ -0,0 +1,339 @@
"""Tests for HERMES_HOME credential-file read blocking in file_safety.
Regression for https://github.com/NousResearch/hermes-agent/issues/17656 —
``read_file`` was previously only sandboxed against ``HERMES_HOME`` itself,
which left ``auth.json`` and ``.anthropic_oauth.json`` (plaintext provider
keys + OAuth tokens) readable by the agent. A prompt-injection reaching
``read_file`` could exfiltrate active credentials.
These tests verify that ``get_read_block_error`` returns a denial message
for the credential stores while leaving arbitrary ``HERMES_HOME`` files
readable, and that the existing ``skills/.hub`` deny still applies.
"""
from __future__ import annotations
import os
from pathlib import Path
import pytest
@pytest.fixture()
def fake_home(tmp_path, monkeypatch):
"""Point ``_hermes_home_path()`` at a tmp dir for isolated checks."""
import agent.file_safety as fs
home = tmp_path / "hermes_home"
home.mkdir()
monkeypatch.setattr(fs, "_hermes_home_path", lambda: home)
return home
def _create(home: Path, rel: str | Path) -> Path:
"""Create the file (with parents) so realpath() resolves it."""
p = home / rel
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text("dummy", encoding="utf-8")
return p
def test_auth_json_blocked(fake_home):
from agent.file_safety import get_read_block_error
auth = _create(fake_home, "auth.json")
err = get_read_block_error(str(auth))
assert err is not None
assert "credential store" in err
assert "auth.json" in err
def test_auth_lock_blocked(fake_home):
from agent.file_safety import get_read_block_error
lock = _create(fake_home, "auth.lock")
err = get_read_block_error(str(lock))
assert err is not None
assert "credential store" in err
def test_anthropic_oauth_json_blocked(fake_home):
from agent.file_safety import get_read_block_error
oauth = _create(fake_home, ".anthropic_oauth.json")
err = get_read_block_error(str(oauth))
assert err is not None
assert "credential store" in err
def test_google_oauth_json_blocked(fake_home):
"""Gemini OAuth tokens live under auth/google_oauth.json — blocked."""
from agent.file_safety import get_read_block_error
oauth = _create(fake_home, Path("auth") / "google_oauth.json")
err = get_read_block_error(str(oauth))
assert err is not None
assert "credential store" in err
def test_arbitrary_hermes_home_file_not_blocked(fake_home):
"""Non-credential files inside HERMES_HOME stay readable."""
from agent.file_safety import get_read_block_error
safe = _create(fake_home, "session_log.txt")
assert get_read_block_error(str(safe)) is None
def test_subdirectory_named_auth_json_not_blocked(fake_home):
"""Only the top-level auth.json is the credential store; a file with the
same name in a subdirectory (e.g., a skill mock) must remain readable."""
from agent.file_safety import get_read_block_error
nested = _create(fake_home, Path("skills") / "my-skill" / "auth.json")
assert get_read_block_error(str(nested)) is None
def test_skills_hub_block_still_applies(fake_home):
"""Regression guard: the original skills/.hub deny must keep working."""
from agent.file_safety import get_read_block_error
hub_file = _create(fake_home, "skills/.hub/manifest.json")
err = get_read_block_error(str(hub_file))
assert err is not None
assert "internal Hermes cache file" in err
def test_path_traversal_resolves_to_blocked(fake_home, tmp_path):
"""A path that traverses through a sibling dir back into HERMES_HOME's
auth.json must still be caught — the check resolves through realpath."""
from agent.file_safety import get_read_block_error
_create(fake_home, "auth.json")
sibling = tmp_path / "elsewhere"
sibling.mkdir()
traversal = sibling / ".." / "hermes_home" / "auth.json"
err = get_read_block_error(str(traversal))
assert err is not None
assert "credential store" in err
def test_symlink_to_auth_json_blocked(fake_home, tmp_path):
"""A symlink pointing at HERMES_HOME/auth.json from outside the home
must be blocked — readlink-resolution catches the indirection."""
from agent.file_safety import get_read_block_error
target = _create(fake_home, "auth.json")
link = tmp_path / "shim.json"
try:
os.symlink(target, link)
except (OSError, NotImplementedError):
pytest.skip("symlinks not supported on this platform/filesystem")
err = get_read_block_error(str(link))
assert err is not None
assert "credential store" in err
def test_read_file_tool_blocks_relative_path_under_terminal_cwd(
fake_home, tmp_path, monkeypatch
):
"""Bypass guard: a relative path like ``"auth.json"`` resolved by
``read_file_tool`` against ``TERMINAL_CWD == HERMES_HOME`` must still
be blocked, even though ``get_read_block_error``'s own ``resolve()``
is anchored at the (different) Python process cwd.
"""
import json
import tools.file_tools as ft
_create(fake_home, "auth.json")
# Force the file_tools resolver to anchor relative paths at HERMES_HOME
# while the Python process cwd remains tmp_path (a different directory).
monkeypatch.setenv("TERMINAL_CWD", str(fake_home))
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(
ft, "_get_live_tracking_cwd", lambda task_id="default": None
)
out = json.loads(ft.read_file_tool("auth.json"))
assert "error" in out
assert "credential store" in out["error"]
def test_read_file_tool_blocks_nested_google_oauth_path(
fake_home, tmp_path, monkeypatch
):
"""The real read_file tool must not return Gemini OAuth token material."""
import json
import tools.file_tools as ft
oauth = _create(fake_home, Path("auth") / "google_oauth.json")
oauth.write_text(
json.dumps(
{
"refresh": "REFRESH_TOKEN_MARKER",
"access": "ACCESS_TOKEN_MARKER",
"email": "user@example.com",
}
),
encoding="utf-8",
)
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(
ft, "_get_live_tracking_cwd", lambda task_id="default": None
)
out = json.loads(ft.read_file_tool(str(oauth), task_id="google-oauth-test"))
assert "error" in out
assert "credential store" in out["error"]
assert "REFRESH_TOKEN_MARKER" not in json.dumps(out)
assert "ACCESS_TOKEN_MARKER" not in json.dumps(out)
# ---------------------------------------------------------------------------
# Widening: .env, webhook_subscriptions.json, mcp-tokens/
# ---------------------------------------------------------------------------
def test_dotenv_blocked(fake_home):
""".env in HERMES_HOME holds API keys — blocked."""
from agent.file_safety import get_read_block_error
env = _create(fake_home, ".env")
err = get_read_block_error(str(env))
assert err is not None
assert "credential store" in err
def test_webhook_subscriptions_blocked(fake_home):
"""webhook_subscriptions.json holds per-route HMAC secrets — blocked."""
from agent.file_safety import get_read_block_error
subs = _create(fake_home, "webhook_subscriptions.json")
err = get_read_block_error(str(subs))
assert err is not None
assert "credential store" in err
def test_mcp_tokens_file_blocked(fake_home):
"""Files under mcp-tokens/ hold OAuth tokens — blocked."""
from agent.file_safety import get_read_block_error
tok = _create(fake_home, Path("mcp-tokens") / "github.json")
err = get_read_block_error(str(tok))
assert err is not None
assert "MCP token" in err
def test_mcp_tokens_nested_blocked(fake_home):
"""Nested files inside mcp-tokens/ are also blocked."""
from agent.file_safety import get_read_block_error
tok = _create(fake_home, Path("mcp-tokens") / "providers" / "azure.json")
err = get_read_block_error(str(tok))
assert err is not None
assert "MCP token" in err
def test_mcp_tokens_dir_itself_blocked(fake_home):
"""The mcp-tokens directory itself is blocked (listing is exfiltrating)."""
from agent.file_safety import get_read_block_error
tokens_dir = fake_home / "mcp-tokens"
tokens_dir.mkdir(parents=True, exist_ok=True)
err = get_read_block_error(str(tokens_dir))
assert err is not None
assert "MCP token" in err
def test_identically_named_hermes_files_outside_home_not_blocked(
fake_home, tmp_path
):
"""Hermes-specific filenames (``auth.json``, ``mcp-tokens/``, ``google_oauth.json``)
outside HERMES_HOME must remain readable — the gate is per-location for
those, not per-filename. ``.env`` is the exception: it's blocked anywhere
on disk (see test_project_local_env_blocked) because the basename always
means \"secret-bearing environment file\" regardless of directory."""
from agent.file_safety import get_read_block_error
project = tmp_path / "myproject"
project.mkdir()
# auth.json outside HERMES_HOME — readable (per-location gate).
p = project / "auth.json"
p.write_text("not secret here", encoding="utf-8")
assert get_read_block_error(str(p)) is None, (
"auth.json outside HERMES_HOME should NOT be blocked"
)
google_oauth = project / "auth" / "google_oauth.json"
google_oauth.parent.mkdir()
google_oauth.write_text("not really a token", encoding="utf-8")
assert get_read_block_error(str(google_oauth)) is None
tokens = project / "mcp-tokens"
tokens.mkdir()
tok_file = tokens / "token.json"
tok_file.write_text("not really a token", encoding="utf-8")
assert get_read_block_error(str(tok_file)) is None
def test_non_secret_auth_subtree_file_not_blocked(fake_home):
"""Only the known Google OAuth token path is blocked, not all auth/*."""
from agent.file_safety import get_read_block_error
note = _create(fake_home, Path("auth") / "notes.json")
assert get_read_block_error(str(note)) is None
def test_config_yaml_not_blocked(fake_home):
"""config.yaml is NOT a credential file — agent should still be
able to read it for debugging. (Writes are denied separately by
is_write_denied; reads stay allowed.)"""
from agent.file_safety import get_read_block_error
cfg = _create(fake_home, "config.yaml")
assert get_read_block_error(str(cfg)) is None
def test_profile_mode_blocks_root_credentials(tmp_path, monkeypatch):
"""Under a profile, HERMES_HOME = <root>/profiles/<name>, but
<root>/auth.json must ALSO be blocked — credentials at root are
inherited by every profile."""
import agent.file_safety as fs
root = tmp_path / "hermes"
profile = root / "profiles" / "coder"
profile.mkdir(parents=True)
monkeypatch.setattr(fs, "_hermes_home_path", lambda: profile)
monkeypatch.setattr(fs, "_hermes_root_path", lambda: root)
from agent.file_safety import get_read_block_error
# Profile-local credential store: blocked
profile_auth = profile / "auth.json"
profile_auth.write_text("x")
assert "credential store" in (get_read_block_error(str(profile_auth)) or "")
# Root-level credential store: ALSO blocked (this is the widening)
root_auth = root / "auth.json"
root_auth.write_text("x")
assert "credential store" in (get_read_block_error(str(root_auth)) or "")
# Root-level .env: blocked too
root_env = root / ".env"
root_env.write_text("x")
assert "credential store" in (get_read_block_error(str(root_env)) or "")
# Root-level Google OAuth token store: blocked too
root_google_oauth = root / "auth" / "google_oauth.json"
root_google_oauth.parent.mkdir(parents=True, exist_ok=True)
root_google_oauth.write_text("x")
assert "credential store" in (
get_read_block_error(str(root_google_oauth)) or ""
)
# Root-level mcp-tokens: blocked
root_tok = root / "mcp-tokens" / "gh.json"
root_tok.parent.mkdir(parents=True, exist_ok=True)
root_tok.write_text("x")
assert "MCP token" in (get_read_block_error(str(root_tok)) or "")

218
tests/agent/test_file_safety_cross_profile.py Normal file
View file

@ -0,0 +1,218 @@
"""Tests for the cross-Hermes-profile write guard in agent/file_safety.
The guard fires when a tool tries to write into another Hermes profile's
skills/plugins/cron/memories directory. It's a soft guard — defense in
depth, NOT a security boundary — but it prevents the agent from silently
corrupting a profile that belongs to a different session.
Reference: May 2026 incident — a hermes-security profile session
accidentally edited skills under both ~/.hermes/profiles/hermes-security/skills/
AND ~/.hermes/skills/ (the default profile's skills), realizing only
afterwards that the second path belonged to a different profile.
"""
from __future__ import annotations
from pathlib import Path
import pytest
# ---------------------------------------------------------------------------
# Helpers — set up a fake Hermes root with two profiles, monkeypatch the
# resolver helpers so the classifier sees the test layout.
# ---------------------------------------------------------------------------
@pytest.fixture
def fake_hermes(tmp_path, monkeypatch):
"""Build a fake Hermes layout:
<tmp>/
skills/foo/SKILL.md # default profile
plugins/foo/__init__.py
cron/<state>
memories/MEMORY.md
profiles/
hermes-security/
skills/foo/SKILL.md # named profile
plugins/...
coder/
skills/foo/SKILL.md # another named profile
"""
root = tmp_path / "fake-hermes"
(root / "skills" / "foo").mkdir(parents=True)
(root / "skills" / "foo" / "SKILL.md").write_text("# default skill\n")
(root / "plugins" / "foo").mkdir(parents=True)
(root / "memories").mkdir(parents=True)
(root / "cron").mkdir(parents=True)
sec_home = root / "profiles" / "hermes-security"
(sec_home / "skills" / "foo").mkdir(parents=True)
(sec_home / "skills" / "foo" / "SKILL.md").write_text("# sec skill\n")
(sec_home / "plugins").mkdir(parents=True)
coder_home = root / "profiles" / "coder"
(coder_home / "skills" / "foo").mkdir(parents=True)
(coder_home / "skills" / "foo" / "SKILL.md").write_text("# coder skill\n")
# Monkeypatch the resolver functions used by file_safety so each test
# can choose which profile is "active".
import hermes_constants
monkeypatch.setattr(hermes_constants, "get_default_hermes_root", lambda: root)
# The reloads below ensure get_cross_profile_warning/classify see the patched root.
import agent.file_safety as fs
monkeypatch.setattr(fs, "_hermes_root_path", lambda: root)
return {
"root": root,
"default_home": root,
"security_home": sec_home,
"coder_home": coder_home,
}
def _set_active_home(monkeypatch, hermes_home: Path):
"""Point file_safety._hermes_home_path at a specific profile dir."""
import agent.file_safety as fs
monkeypatch.setattr(fs, "_hermes_home_path", lambda: hermes_home)
# ---------------------------------------------------------------------------
# _resolve_active_profile_name
# ---------------------------------------------------------------------------
class TestResolveActiveProfileName:
def test_default_when_home_is_root(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["default_home"])
from agent.file_safety import _resolve_active_profile_name
assert _resolve_active_profile_name() == "default"
def test_named_profile(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import _resolve_active_profile_name
assert _resolve_active_profile_name() == "hermes-security"
def test_falls_back_to_default_on_resolution_failure(self, fake_hermes, monkeypatch):
"""If HERMES_HOME resolution raises, return 'default' rather than crashing the tool."""
import agent.file_safety as fs
def _boom():
raise RuntimeError("simulated")
monkeypatch.setattr(fs, "_hermes_home_path", _boom)
# Should not raise — falls back to "default"
assert fs._resolve_active_profile_name() == "default"
# ---------------------------------------------------------------------------
# classify_cross_profile_target
# ---------------------------------------------------------------------------
class TestClassifyCrossProfileTarget:
def test_same_profile_write_returns_none(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
result = classify_cross_profile_target(
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
)
assert result is None
def test_security_writing_default_skill(self, fake_hermes, monkeypatch):
"""The exact incident from May 2026."""
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
result = classify_cross_profile_target(
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
)
assert result is not None
assert result["active_profile"] == "hermes-security"
assert result["target_profile"] == "default"
assert result["area"] == "skills"
def test_default_writing_security_skill(self, fake_hermes, monkeypatch):
"""Inverse direction — default-profile session reaching into a named profile."""
_set_active_home(monkeypatch, fake_hermes["default_home"])
from agent.file_safety import classify_cross_profile_target
result = classify_cross_profile_target(
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
)
assert result is not None
assert result["active_profile"] == "default"
assert result["target_profile"] == "hermes-security"
def test_named_to_named_cross_profile(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
result = classify_cross_profile_target(
str(fake_hermes["coder_home"] / "skills" / "foo" / "SKILL.md")
)
assert result is not None
assert result["target_profile"] == "coder"
@pytest.mark.parametrize("area", ["skills", "plugins", "cron", "memories"])
def test_all_profile_scoped_areas_classified(self, fake_hermes, monkeypatch, area):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
target = fake_hermes["default_home"] / area / "foo.txt"
result = classify_cross_profile_target(str(target))
assert result is not None
assert result["area"] == area
def test_non_hermes_path_returns_none(self, fake_hermes, monkeypatch, tmp_path):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
# Path outside any Hermes root
assert classify_cross_profile_target(str(tmp_path / "random.txt")) is None
def test_hermes_config_not_classified_as_cross_profile(self, fake_hermes, monkeypatch):
"""Files under <root>/config.yaml or <root>/.env are NOT profile-scoped
(already covered by build_write_denied_paths). Don't double-warn."""
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import classify_cross_profile_target
# config.yaml at root level is not in PROFILE_SCOPED_AREAS
result = classify_cross_profile_target(
str(fake_hermes["default_home"] / "config.yaml")
)
assert result is None
# ---------------------------------------------------------------------------
# get_cross_profile_warning
# ---------------------------------------------------------------------------
class TestGetCrossProfileWarning:
def test_in_profile_returns_none(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import get_cross_profile_warning
assert get_cross_profile_warning(
str(fake_hermes["security_home"] / "skills" / "foo" / "SKILL.md")
) is None
def test_cross_profile_warning_names_both_profiles(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import get_cross_profile_warning
warn = get_cross_profile_warning(
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
)
assert warn is not None
# Must name BOTH profiles so the model knows which is which.
assert "default" in warn
assert "hermes-security" in warn
# Must name the bypass kwarg.
assert "cross_profile=True" in warn
# Must reference the area.
assert "skills" in warn
def test_warning_is_defense_in_depth_not_boundary(self, fake_hermes, monkeypatch):
_set_active_home(monkeypatch, fake_hermes["security_home"])
from agent.file_safety import get_cross_profile_warning
warn = get_cross_profile_warning(
str(fake_hermes["default_home"] / "skills" / "foo" / "SKILL.md")
)
# Must self-document as defense-in-depth so future reviewers
# don't promote it to a hard block.
assert "not a security boundary" in warn.lower()

224
tests/agent/test_file_safety_sandbox_mirror.py Normal file
View file

@ -0,0 +1,224 @@
"""Tests for the sandbox-mirror write guard in agent/file_safety.
The guard fires when a tool tries to write into the per-task mirror
directory created by a non-local terminal backend (Docker, Daytona, etc.).
Those paths look like ``…/sandboxes/<backend>/<task>/home/.hermes/…`` and
they accumulate divergent copies of authoritative profile state (SOUL.md,
config.yaml, memories/*.md) because the host Hermes process never reads
them. Soft guard — defense in depth, NOT a security boundary.
Reference: #32049 — under ``terminal.backend: docker``, the agent's
``write_file`` / ``patch`` calls landed on the sandbox mirror of SOUL.md
while the host process kept loading the untouched authoritative file.
The agent reported success; the rule never took effect.
"""
from __future__ import annotations
from pathlib import Path
import pytest
# ---------------------------------------------------------------------------
# classify_sandbox_mirror_target — pure path-shape detection
# ---------------------------------------------------------------------------
class TestClassifySandboxMirrorTarget:
def test_docker_mirror_soul_md_classified(self, tmp_path):
"""The exact path shape reported in #32049."""
from agent.file_safety import classify_sandbox_mirror_target
target = (
tmp_path
/ "profiles" / "group1"
/ "sandboxes" / "docker" / "default" / "home" / ".hermes"
/ "profiles" / "group1" / "SOUL.md"
)
target.parent.mkdir(parents=True)
target.write_text("# mirror copy\n")
result = classify_sandbox_mirror_target(str(target))
assert result is not None
assert result["target_path"] == str(target.resolve())
assert result["mirror_root"].endswith(
"sandboxes/docker/default/home/.hermes"
)
assert result["inner_path"] == "profiles/group1/SOUL.md"
@pytest.mark.parametrize(
"backend,inner",
[
("docker", "profiles/coder/memories/MEMORY.md"),
("daytona", "profiles/default/cron/jobs.json"),
("podman", ".env"),
],
)
def test_other_backends_and_inner_files_match(self, tmp_path, backend, inner):
"""The detector is backend-agnostic — sandbox-mirror shape is what matters."""
from agent.file_safety import classify_sandbox_mirror_target
target = (
tmp_path
/ "sandboxes" / backend / "task-42" / "home" / ".hermes"
/ Path(inner)
)
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text("x")
result = classify_sandbox_mirror_target(str(target))
assert result is not None
assert result["inner_path"] == inner
assert backend in result["mirror_root"]
def test_path_outside_sandbox_returns_none(self, tmp_path):
"""A plain Hermes path is not a mirror."""
from agent.file_safety import classify_sandbox_mirror_target
target = tmp_path / ".hermes" / "profiles" / "group1" / "SOUL.md"
target.parent.mkdir(parents=True)
target.write_text("# real SOUL\n")
assert classify_sandbox_mirror_target(str(target)) is None
def test_sandboxes_segment_without_home_hermes_returns_none(self, tmp_path):
"""A ``sandboxes/`` directory unrelated to Hermes-state mirroring (e.g.
the sandbox workspace itself) is not flagged."""
from agent.file_safety import classify_sandbox_mirror_target
target = (
tmp_path
/ "sandboxes" / "docker" / "task-42" / "workspace" / "main.py"
)
target.parent.mkdir(parents=True)
target.write_text("print('hi')\n")
assert classify_sandbox_mirror_target(str(target)) is None
def test_sandboxes_segment_with_home_but_no_hermes_returns_none(self, tmp_path):
"""``sandboxes/<backend>/<task>/home/anything-not-hermes`` is not a mirror."""
from agent.file_safety import classify_sandbox_mirror_target
target = (
tmp_path
/ "sandboxes" / "docker" / "task-42" / "home" / ".bashrc"
)
target.parent.mkdir(parents=True)
target.write_text("alias ll='ls -la'\n")
assert classify_sandbox_mirror_target(str(target)) is None
def test_truncated_sandbox_path_returns_none(self, tmp_path):
"""``…/sandboxes/<backend>/<task>`` without ``home/.hermes/<thing>`` is not a mirror."""
from agent.file_safety import classify_sandbox_mirror_target
target = tmp_path / "sandboxes" / "docker" / "task-42"
target.mkdir(parents=True)
assert classify_sandbox_mirror_target(str(target)) is None
def test_non_existent_path_still_classifies_by_shape(self, tmp_path):
"""Detection is path-shape only — it must not require the file to exist
(the agent is about to CREATE the mirror file, that's the bug)."""
from agent.file_safety import classify_sandbox_mirror_target
target = (
tmp_path
/ "profiles" / "group1"
/ "sandboxes" / "docker" / "default" / "home" / ".hermes"
/ "profiles" / "group1" / "SOUL.md"
)
# Parent directory exists so .resolve() doesn't strip the tail
# under strict mode, but the file itself does NOT exist.
target.parent.mkdir(parents=True)
assert not target.exists()
result = classify_sandbox_mirror_target(str(target))
assert result is not None
assert result["inner_path"] == "profiles/group1/SOUL.md"
# ---------------------------------------------------------------------------
# get_sandbox_mirror_warning — the model-facing string
# ---------------------------------------------------------------------------
class TestGetSandboxMirrorWarning:
def test_non_mirror_returns_none(self, tmp_path):
from agent.file_safety import get_sandbox_mirror_warning
target = tmp_path / ".hermes" / "profiles" / "group1" / "SOUL.md"
target.parent.mkdir(parents=True)
target.write_text("# real SOUL\n")
assert get_sandbox_mirror_warning(str(target)) is None
def test_mirror_warning_names_mirror_root_and_inner_path(self, tmp_path):
from agent.file_safety import get_sandbox_mirror_warning
target = (
tmp_path
/ "profiles" / "group1"
/ "sandboxes" / "docker" / "default" / "home" / ".hermes"
/ "profiles" / "group1" / "SOUL.md"
)
target.parent.mkdir(parents=True)
target.write_text("# mirror copy\n")
warn = get_sandbox_mirror_warning(str(target))
assert warn is not None
# Must name the mirror root so the user can locate the sandbox.
assert "sandboxes/docker/default/home/.hermes" in warn
# Must hint at what the agent likely meant.
assert "profiles/group1/SOUL.md" in warn
# Must name the bypass kwarg shared with the cross-profile guard.
assert "cross_profile=True" in warn
def test_warning_is_defense_in_depth_not_boundary(self, tmp_path):
from agent.file_safety import get_sandbox_mirror_warning
target = (
tmp_path
/ "sandboxes" / "docker" / "t" / "home" / ".hermes"
/ "profiles" / "g" / "SOUL.md"
)
target.parent.mkdir(parents=True)
target.write_text("x")
warn = get_sandbox_mirror_warning(str(target))
# Must self-document as defense-in-depth so future reviewers
# don't promote it to a hard block (matches the existing
# cross-profile guard's contract).
assert "not a security boundary" in warn.lower()
# ---------------------------------------------------------------------------
# Independence from cross-profile classifier
# ---------------------------------------------------------------------------
class TestSandboxMirrorIsOrthogonalToCrossProfile:
"""The sandbox-mirror guard must fire even when the inner path is
in-profile from the host's view — the bug is the mirror, not the
profile mismatch."""
def test_same_profile_mirror_still_flagged(self, tmp_path, monkeypatch):
import agent.file_safety as fs
monkeypatch.setattr(fs, "_hermes_root_path", lambda: tmp_path)
monkeypatch.setattr(fs, "_hermes_home_path", lambda: tmp_path / "profiles" / "group1")
target = (
tmp_path
/ "profiles" / "group1"
/ "sandboxes" / "docker" / "default" / "home" / ".hermes"
/ "profiles" / "group1" / "SOUL.md"
)
target.parent.mkdir(parents=True)
target.write_text("x")
# cross-profile classifier: active profile == target's inner-mirror
# profile name; on the existing detector the path's parts[2] is
# ``sandboxes``, not a scoped area, so it returns None.
assert fs.classify_cross_profile_target(str(target)) is None
# sandbox-mirror classifier: fires unconditionally on the shape.
assert fs.classify_sandbox_mirror_target(str(target)) is not None

2
tests/agent/test_gemini_cloudcode.py
View file

@ -18,8 +18,6 @@ import json
import stat
import time
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest

1
tests/agent/test_gemini_free_tier_gate.py
View file

@ -3,7 +3,6 @@ from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
from agent.gemini_native_adapter import (
gemini_http_error,

24
tests/agent/test_gemini_native_adapter.py
View file

@ -326,3 +326,27 @@ def test_stream_event_translation_keeps_identical_calls_in_distinct_parts():
assert tool_chunks[0].choices[0].delta.tool_calls[0].index == 0
assert tool_chunks[1].choices[0].delta.tool_calls[0].index == 1
assert tool_chunks[0].choices[0].delta.tool_calls[0].id != tool_chunks[1].choices[0].delta.tool_calls[0].id
def test_max_tokens_none_defaults_to_gemini_output_ceiling():
"""max_tokens=None must send the model's full output ceiling, not omit it.
Gemini's native generateContent applies a low internal default when
maxOutputTokens is absent, truncating tool calls mid-stream. Hermes passes
None to mean "unlimited", so the adapter must translate that to the
published 65,535 ceiling rather than leaving the field unset.
"""
from agent.gemini_native_adapter import (
build_gemini_request,
GEMINI_DEFAULT_MAX_OUTPUT_TOKENS,
)
req = build_gemini_request(messages=[{"role": "user", "content": "hi"}], max_tokens=None)
assert req["generationConfig"]["maxOutputTokens"] == GEMINI_DEFAULT_MAX_OUTPUT_TOKENS == 65535
def test_explicit_max_tokens_is_respected():
from agent.gemini_native_adapter import build_gemini_request
req = build_gemini_request(messages=[{"role": "user", "content": "hi"}], max_tokens=4096)
assert req["generationConfig"]["maxOutputTokens"] == 4096

60
tests/agent/test_i18n.py
View file

@ -167,3 +167,63 @@ def test_t_missing_key_in_non_english_falls_back_to_english(tmp_path, monkeypatc
def test_t_unknown_language_uses_english():
"""Unknown lang codes normalize to English, not to a key-path fallback."""
assert i18n.t("approval.denied", lang="klingon") == i18n.t("approval.denied", lang="en")
# ---------------------------------------------------------------------------
# _locales_dir resolution ladder -- regression for #23943 / #27632 / #35374.
# Sealed installs (Nix store venv, pip wheel) have no source tree next to
# agent/, so _locales_dir must resolve via env override or the data scheme.
# ---------------------------------------------------------------------------
def test_locales_dir_env_override_used_when_dir_exists(tmp_path, monkeypatch):
"""HERMES_BUNDLED_LOCALES wins when it points at a real directory."""
bundled = tmp_path / "bundled-locales"
bundled.mkdir()
monkeypatch.setenv("HERMES_BUNDLED_LOCALES", str(bundled))
assert i18n._locales_dir() == bundled
def test_locales_dir_env_override_ignored_when_missing(tmp_path, monkeypatch):
"""A bogus HERMES_BUNDLED_LOCALES falls through to source/wheel resolution
instead of returning a path that doesn't exist."""
monkeypatch.setenv("HERMES_BUNDLED_LOCALES", str(tmp_path / "does-not-exist"))
result = i18n._locales_dir()
assert result != tmp_path / "does-not-exist"
# In a source checkout this is the repo-root locales dir.
assert result.name == "locales"
def test_locales_dir_falls_back_to_data_scheme(tmp_path, monkeypatch):
"""When neither the env override nor a source-adjacent locales/ exists,
_locales_dir uses sysconfig's data scheme (the pip-wheel layout)."""
import sysconfig
# No env override.
monkeypatch.delenv("HERMES_BUNDLED_LOCALES", raising=False)
# Force the source-adjacent path to a location with no locales/ dir.
fake_pkg = tmp_path / "site-packages" / "agent"
fake_pkg.mkdir(parents=True)
monkeypatch.setattr(i18n, "__file__", str(fake_pkg / "i18n.py"))
# Stand up a fake data scheme containing locales/.
data_root = tmp_path / "data-scheme"
(data_root / "locales").mkdir(parents=True)
real_get_path = sysconfig.get_path
def fake_get_path(name, *args, **kwargs):
if name == "data":
return str(data_root)
return real_get_path(name, *args, **kwargs)
monkeypatch.setattr(i18n.sysconfig, "get_path", fake_get_path)
assert i18n._locales_dir() == data_root / "locales"
def test_t_resolves_real_string_in_source_checkout():
"""Sanity: in the test environment (a source checkout) t() must return a
human string, never the bare key path. Guards against catalog-load
regressions independent of packaging."""
assert i18n.t("gateway.reset.header_default", lang="en") != "gateway.reset.header_default"
assert i18n.t("gateway.status.header", lang="en") != "gateway.status.header"

189
tests/agent/test_image_routing.py
View file

@ -6,7 +6,6 @@ import base64
from pathlib import Path
from unittest.mock import patch
import pytest
from agent.image_routing import (
_coerce_capability_bool,
@ -16,6 +15,7 @@ from agent.image_routing import (
_supports_vision_override,
build_native_content_parts,
decide_image_input_mode,
extract_image_refs,
)
@ -449,3 +449,190 @@ class TestLargeImageHandling:
assert len(parts) == 2
assert parts[0]["type"] == "text"
assert parts[1]["type"] == "image_url"
# ─── extract_image_refs ──────────────────────────────────────────────────────
class TestExtractImageRefs:
"""Scan task body / inbound text for image paths and URLs (kanban worker
enrichment, issue raised May 2026)."""
def test_empty_or_none_returns_empty(self):
assert extract_image_refs("") == ([], [])
assert extract_image_refs(None) == ([], []) # type: ignore[arg-type]
def test_finds_absolute_path(self, tmp_path: Path):
img = tmp_path / "screenshot.png"
img.write_bytes(_png_bytes())
body = f"Look at {img} and tell me what's wrong."
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == []
def test_finds_home_relative_path(self, tmp_path: Path, monkeypatch):
# Simulate ~/foo.png by pointing HOME at tmp_path and creating the file
monkeypatch.setenv("HOME", str(tmp_path))
img = tmp_path / "foo.png"
img.write_bytes(_png_bytes())
paths, urls = extract_image_refs("see ~/foo.png please")
assert paths == [str(img)]
assert urls == []
def test_skips_nonexistent_paths(self, tmp_path: Path):
# Path-shaped but no file on disk → skipped.
body = f"What's at {tmp_path}/never_created.png ?"
paths, urls = extract_image_refs(body)
assert paths == []
assert urls == []
def test_finds_http_image_url(self):
body = "Check out https://example.com/photos/cat.png — cute right?"
paths, urls = extract_image_refs(body)
assert paths == []
assert urls == ["https://example.com/photos/cat.png"]
def test_finds_https_url_with_query_string(self):
body = "Diagram: https://cdn.example.com/img.jpeg?size=large&v=2 here"
paths, urls = extract_image_refs(body)
assert urls == ["https://cdn.example.com/img.jpeg?size=large&v=2"]
def test_url_trailing_punctuation_stripped(self):
# Prose punctuation right after the URL must not be part of the URL.
body = "See https://example.com/a.png."
paths, urls = extract_image_refs(body)
assert urls == ["https://example.com/a.png"]
def test_ignores_non_image_urls(self):
body = "See https://example.com/page.html and https://x.com/y.pdf"
paths, urls = extract_image_refs(body)
assert urls == []
def test_dedupes_paths_and_urls(self, tmp_path: Path):
img = tmp_path / "dup.png"
img.write_bytes(_png_bytes())
body = (
f"First {img} then again {img}. "
"Also https://example.com/x.png and https://example.com/x.png again."
)
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == ["https://example.com/x.png"]
def test_ignores_paths_in_fenced_code_block(self, tmp_path: Path):
img = tmp_path / "real.png"
img.write_bytes(_png_bytes())
body = (
"Outside the block, attach this:\n"
f"{img}\n"
"But not these examples:\n"
"```\n"
f"some_other_image: /tmp/example.png\n"
f"url: https://example.com/example.png\n"
"```\n"
)
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == []
def test_ignores_paths_in_inline_code(self, tmp_path: Path):
img = tmp_path / "real.jpg"
img.write_bytes(_png_bytes())
body = (
f"Attach {img}, but ignore the example "
"`https://example.com/skip.png` in backticks."
)
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == []
def test_does_not_match_paths_inside_urls(self, tmp_path: Path):
# The lookbehind in the regex prevents matching the path-portion of
# a URL as a local path. Only the URL should be detected.
body = "Just the URL: https://example.com/some/dir/image.png"
paths, urls = extract_image_refs(body)
assert paths == []
assert urls == ["https://example.com/some/dir/image.png"]
def test_mixed_paths_and_urls(self, tmp_path: Path):
img = tmp_path / "local.png"
img.write_bytes(_png_bytes())
body = (
f"Compare local {img} against the design at "
"https://example.com/design/v2.png — does it match?"
)
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
assert urls == ["https://example.com/design/v2.png"]
def test_case_insensitive_extension(self, tmp_path: Path):
img = tmp_path / "shouty.PNG"
img.write_bytes(_png_bytes())
body = f"see {img}"
paths, urls = extract_image_refs(body)
assert paths == [str(img)]
# ─── build_native_content_parts with URLs ────────────────────────────────────
class TestBuildNativeContentPartsURLs:
"""URL pass-through support added so kanban task bodies (and other
inbound surfaces) can route remote image URLs straight to the model."""
def test_url_only_no_local_paths(self):
parts, skipped = build_native_content_parts(
"what is this?",
[],
image_urls=["https://example.com/diagram.png"],
)
assert skipped == []
assert len(parts) == 2
assert parts[0]["type"] == "text"
assert "[Image attached: https://example.com/diagram.png]" in parts[0]["text"]
assert parts[0]["text"].startswith("what is this?")
assert parts[1] == {
"type": "image_url",
"image_url": {"url": "https://example.com/diagram.png"},
}
def test_mixed_path_and_url(self, tmp_path: Path):
img = tmp_path / "local.png"
img.write_bytes(_png_bytes())
parts, skipped = build_native_content_parts(
"compare these",
[str(img)],
image_urls=["https://example.com/remote.jpg"],
)
assert skipped == []
# 1 text + 2 image parts (local data URL first, then remote URL).
image_parts = [p for p in parts if p.get("type") == "image_url"]
assert len(image_parts) == 2
assert image_parts[0]["image_url"]["url"].startswith("data:image/png;base64,")
assert image_parts[1]["image_url"]["url"] == "https://example.com/remote.jpg"
text = parts[0]["text"]
assert "[Image attached at:" in text
assert "[Image attached: https://example.com/remote.jpg]" in text
def test_empty_url_list_is_no_op(self, tmp_path: Path):
img = tmp_path / "x.png"
img.write_bytes(_png_bytes())
# image_urls=[] should behave the same as not passing it at all.
parts_no_urls, _ = build_native_content_parts("hi", [str(img)])
parts_empty_urls, _ = build_native_content_parts("hi", [str(img)], image_urls=[])
assert parts_no_urls == parts_empty_urls
def test_blank_url_strings_are_dropped(self):
parts, _ = build_native_content_parts(
"x", [], image_urls=["", " ", "https://example.com/a.png"]
)
image_parts = [p for p in parts if p.get("type") == "image_url"]
assert len(image_parts) == 1
assert image_parts[0]["image_url"]["url"] == "https://example.com/a.png"
def test_url_only_inserts_default_prompt_when_text_empty(self):
parts, _ = build_native_content_parts(
"", [], image_urls=["https://example.com/a.png"]
)
assert parts[0]["type"] == "text"
assert parts[0]["text"].startswith("What do you see in this image?")

9
tests/agent/test_insights.py
View file

@ -2,16 +2,16 @@
import time
import pytest
from pathlib import Path
from hermes_state import SessionDB
from agent.insights import (
InsightsEngine,
_estimate_cost,
_format_duration,
_bar_chart,
_has_known_pricing,
_DEFAULT_PRICING,
)
from agent.usage_pricing import (
format_duration_compact as _format_duration,
has_known_pricing as _has_known_pricing,
)
@ -596,7 +596,6 @@ class TestEdgeCases:
def test_tool_usage_from_tool_calls_json(self, db):
"""Tool usage should be extracted from tool_calls JSON when tool_name is NULL."""
import json as _json
db.create_session(session_id="s1", source="cli", model="test")
# Assistant message with tool_calls (this is what CLI produces)
db.append_message("s1", role="assistant", content="Let me search",

25
tests/agent/test_jiter_preload.py Normal file
View file

@ -0,0 +1,25 @@
from __future__ import annotations
import importlib
import sys
from agent import jiter_preload
def test_preload_jiter_native_extension_loads_sdk_parser_dependency():
assert jiter_preload.preload_jiter_native_extension() is True
assert "jiter.jiter" in sys.modules
def test_preload_jiter_native_extension_is_best_effort(monkeypatch):
monkeypatch.setattr(jiter_preload, "_JITER_PRELOADED", False)
def _raise_missing(name: str):
assert name == "jiter.jiter"
raise ModuleNotFoundError(name)
monkeypatch.setattr(importlib, "import_module", _raise_missing)
assert jiter_preload.preload_jiter_native_extension() is False
assert jiter_preload._JITER_PRELOADED is False
assert isinstance(jiter_preload._JITER_PRELOAD_ERROR, ModuleNotFoundError)

22
tests/agent/test_last_total_tokens.py Normal file
View file

@ -0,0 +1,22 @@
"""Test that last_total_tokens is correctly set by ContextCompressor."""
from agent.context_compressor import ContextCompressor
def test_update_from_response_sets_total_tokens():
"""ABC contract: last_total_tokens must be set from API response."""
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30, "total_tokens": 130})
assert c.last_total_tokens == 130
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30})
assert c.last_total_tokens == 130
def test_session_reset_clears_total_tokens():
"""on_session_reset must zero total_tokens."""
c = ContextCompressor(model="test", quiet_mode=True, config_context_length=200000)
c.update_from_response({"prompt_tokens": 100, "completion_tokens": 30, "total_tokens": 130})
c.on_session_reset()
assert c.last_total_tokens == 0

10
tests/agent/test_local_stream_timeout.py
View file

@ -98,6 +98,16 @@ class TestIsLocalEndpoint:
def test_container_dns_names(self, url):
assert is_local_endpoint(url) is True
@pytest.mark.parametrize("url", [
"http://ollama:11434",
"http://litellm:4000/v1",
"http://hermes-litellm:8080",
"http://vllm:8000",
])
def test_unqualified_docker_hostnames(self, url):
"""Unqualified hostnames (no dots) are local — Docker Compose, /etc/hosts, etc."""
assert is_local_endpoint(url) is True
@pytest.mark.parametrize("url", [
"https://api.openai.com",
"https://openrouter.ai/api",

138
tests/agent/test_memory_async_sync.py Normal file
View file

@ -0,0 +1,138 @@
"""Regression guard: end-of-turn memory sync must not block the turn.
Before this fix, ``MemoryManager.sync_all`` / ``queue_prefetch_all`` looped
``provider.sync_turn`` / ``provider.queue_prefetch`` INLINE on the
turn-completion path. A provider making a blocking network/daemon call (a
misconfigured Hindsight daemon was observed blocking ~298s before failing)
held ``run_conversation`` open long after the user saw their response, so
every interface (CLI, TUI, gateway) kept the agent marked "running" for
minutes and any follow-up message triggered an aggressive interrupt that
dropped the message.
The fix dispatches provider work to a single-worker background executor.
``sync_all`` / ``queue_prefetch_all`` return immediately; the work completes
(or fails, logged) in the background. ``flush_pending`` provides a barrier
for session boundaries and deterministic tests. ``shutdown_all`` drains the
executor with a bounded timeout so a wedged provider can't hang teardown.
"""
import time
import pytest
from agent.memory_provider import MemoryProvider
from agent.memory_manager import MemoryManager
class _SlowProvider(MemoryProvider):
"""Provider whose sync/prefetch block, simulating a slow backend."""
_name = "slow"
def __init__(self, delay: float = 1.0):
self._delay = delay
self.sync_done = False
self.prefetch_done = False
@property
def name(self) -> str:
return self._name
def initialize(self, session_id: str = "", **kwargs) -> None:
pass
def is_available(self) -> bool:
return True
def system_prompt_block(self) -> str:
return ""
def prefetch(self, query, *, session_id: str = "") -> str:
return ""
def queue_prefetch(self, query, *, session_id: str = "") -> None:
time.sleep(self._delay)
self.prefetch_done = True
def sync_turn(self, user_content, assistant_content, *, session_id: str = "", messages=None) -> None:
time.sleep(self._delay)
self.sync_done = True
def get_tool_schemas(self):
return []
def handle_tool_call(self, tool_name, args, **kwargs) -> str:
return ""
def test_sync_all_does_not_block_on_slow_provider():
"""The crux of the fix: a slow provider must NOT stall the caller."""
mgr = MemoryManager()
mgr.add_provider(_SlowProvider(delay=2.0))
t0 = time.time()
mgr.sync_all("hi", "hey", session_id="s1")
mgr.queue_prefetch_all("hi", session_id="s1")
elapsed = time.time() - t0
# Provider blocks 2s per call inline; off-thread dispatch returns ~instantly.
assert elapsed < 0.5, f"turn-completion path blocked {elapsed:.2f}s"
def test_background_work_still_completes():
"""Dispatching off-thread must not silently drop the write."""
mgr = MemoryManager()
p = _SlowProvider(delay=0.1)
mgr.add_provider(p)
mgr.sync_all("hi", "hey", session_id="s1")
mgr.queue_prefetch_all("hi", session_id="s1")
assert mgr.flush_pending(timeout=10) is True
assert p.sync_done is True
assert p.prefetch_done is True
def test_flush_pending_no_executor_is_true():
"""flush_pending must be a no-op (return True) before any sync ran."""
mgr = MemoryManager()
assert mgr.flush_pending(timeout=1) is True
def test_no_providers_does_not_create_executor():
"""Builtin-only / no-provider sessions must not spawn an executor."""
mgr = MemoryManager()
mgr.sync_all("hi", "hey")
mgr.queue_prefetch_all("hi")
assert mgr._sync_executor is None
def test_shutdown_all_is_bounded_with_wedged_provider():
"""A provider that never returns must not hang teardown."""
mgr = MemoryManager()
mgr.add_provider(_SlowProvider(delay=30.0))
mgr.sync_all("hi", "hey")
t0 = time.time()
mgr.shutdown_all()
elapsed = time.time() - t0
# Bounded by _SYNC_DRAIN_TIMEOUT_S (5s) plus a little slack.
assert elapsed < 8.0, f"shutdown blocked {elapsed:.1f}s on wedged provider"
def test_writes_are_serialized_in_order():
"""Single-worker executor must preserve turn ordering (N before N+1)."""
order = []
class _OrderProvider(_SlowProvider):
_name = "order"
def sync_turn(self, user_content, assistant_content, *, session_id="", messages=None):
order.append(user_content)
mgr = MemoryManager()
mgr.add_provider(_OrderProvider(delay=0.0))
for i in range(5):
mgr.sync_all(f"turn-{i}", "resp", session_id="s1")
assert mgr.flush_pending(timeout=10) is True
assert order == [f"turn-{i}" for i in range(5)]

365
tests/agent/test_memory_provider.py
View file

@ -2,7 +2,7 @@
import json
import pytest
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
from agent.memory_provider import MemoryProvider
from agent.memory_manager import MemoryManager
@ -84,6 +84,13 @@ class MetadataMemoryProvider(FakeMemoryProvider):
self.memory_writes.append((action, target, content, metadata or {}))
class MessagesMemoryProvider(FakeMemoryProvider):
"""Provider that opts into completed-turn message context."""
def sync_turn(self, user_content, assistant_content, *, session_id="", messages=None):
self.synced_turns.append((user_content, assistant_content, session_id, messages))
# ---------------------------------------------------------------------------
# MemoryProvider ABC tests
# ---------------------------------------------------------------------------
@ -222,6 +229,7 @@ class TestMemoryManager:
mgr.add_provider(p2)
mgr.queue_prefetch_all("next turn")
mgr.flush_pending(timeout=5)
assert p1.queued_prefetches == ["next turn"]
assert p2.queued_prefetches == ["next turn"]
@ -233,9 +241,32 @@ class TestMemoryManager:
mgr.add_provider(p2)
mgr.sync_all("user msg", "assistant msg")
mgr.flush_pending(timeout=5)
assert p1.synced_turns == [("user msg", "assistant msg")]
assert p2.synced_turns == [("user msg", "assistant msg")]
def test_sync_all_passes_messages_to_opted_in_provider(self):
mgr = MemoryManager()
p = MessagesMemoryProvider("external")
mgr.add_provider(p)
messages = [
{"role": "assistant", "tool_calls": [{"id": "call-1"}]},
{"role": "tool", "tool_call_id": "call-1", "content": "ok"},
]
mgr.sync_all("user msg", "assistant msg", session_id="sess-1", messages=messages)
mgr.flush_pending(timeout=5)
assert p.synced_turns == [("user msg", "assistant msg", "sess-1", messages)]
def test_sync_all_omits_messages_for_legacy_provider(self):
mgr = MemoryManager()
p = FakeMemoryProvider("external")
mgr.add_provider(p)
mgr.sync_all("user msg", "assistant msg", messages=[{"role": "tool"}])
mgr.flush_pending(timeout=5)
assert p.synced_turns == [("user msg", "assistant msg")]
def test_sync_failure_doesnt_block_others(self):
"""If one provider's sync fails, others still run."""
mgr = MemoryManager()
@ -246,6 +277,7 @@ class TestMemoryManager:
mgr.add_provider(p2)
mgr.sync_all("user", "assistant")
mgr.flush_pending(timeout=5)
# p1 failed but p2 still synced
assert p2.synced_turns == [("user", "assistant")]
@ -433,7 +465,7 @@ class TestUserInstalledProviderDiscovery:
def test_discover_finds_user_plugins(self, tmp_path, monkeypatch):
"""discover_memory_providers() includes user-installed plugins."""
from plugins.memory import discover_memory_providers, _get_user_plugins_dir
from plugins.memory import discover_memory_providers
self._make_user_memory_plugin(tmp_path, "myexternal")
monkeypatch.setattr(
"plugins.memory._get_user_plugins_dir",
@ -504,6 +536,147 @@ class TestUserInstalledProviderDiscovery:
names = [n for n, _, _ in providers]
assert "notmemory" not in names
def test_load_user_plugin_with_relative_import(self, tmp_path, monkeypatch):
"""User plugins may import sibling modules with relative imports.
Regression: _load_provider_from_dir() imports user plugins under the
synthetic ``_hermes_user_memory.<name>`` package but never registered
that parent namespace in sys.modules, so any relative import inside
the plugin raised
``ModuleNotFoundError: No module named '_hermes_user_memory'``.
"""
from plugins.memory import load_memory_provider
plugin_dir = tmp_path / "plugins" / "relimport"
plugin_dir.mkdir(parents=True)
(plugin_dir / "helper.py").write_text("PROVIDER_NAME = 'relimport'\n")
(plugin_dir / "__init__.py").write_text(
"from agent.memory_provider import MemoryProvider\n"
"from . import helper\n"
"class MyProvider(MemoryProvider):\n"
" @property\n"
" def name(self): return helper.PROVIDER_NAME\n"
" def is_available(self): return True\n"
" def initialize(self, **kw): pass\n"
" def sync_turn(self, *a, **kw): pass\n"
" def get_tool_schemas(self): return []\n"
" def handle_tool_call(self, *a, **kw): return '{}'\n"
)
monkeypatch.setattr(
"plugins.memory._get_user_plugins_dir",
lambda: tmp_path / "plugins",
)
p = load_memory_provider("relimport")
assert p is not None
assert p.name == "relimport"
def test_load_user_plugin_with_nested_subpackage(self, tmp_path, monkeypatch):
"""User plugins may keep their implementation in a nested subpackage.
Plugin repos that target several runtimes commonly expose a thin root
``__init__.py`` re-exporting from a deeper package, and the
intermediate directory may be a namespace package (no __init__.py).
Both must resolve through the synthetic parent namespace.
"""
from plugins.memory import load_memory_provider
plugin_dir = tmp_path / "plugins" / "nestedimpl"
impl_dir = plugin_dir / "adapters" / "hermes" # adapters/ has no __init__.py
impl_dir.mkdir(parents=True)
(impl_dir / "__init__.py").write_text(
"from agent.memory_provider import MemoryProvider\n"
"class MyProvider(MemoryProvider):\n"
" @property\n"
" def name(self): return 'nestedimpl'\n"
" def is_available(self): return True\n"
" def initialize(self, **kw): pass\n"
" def sync_turn(self, *a, **kw): pass\n"
" def get_tool_schemas(self): return []\n"
" def handle_tool_call(self, *a, **kw): return '{}'\n"
)
(plugin_dir / "__init__.py").write_text(
"from .adapters.hermes import MyProvider\n"
"def register(ctx):\n"
" ctx.register_memory_provider(MyProvider())\n"
)
monkeypatch.setattr(
"plugins.memory._get_user_plugins_dir",
lambda: tmp_path / "plugins",
)
p = load_memory_provider("nestedimpl")
assert p is not None
assert p.name == "nestedimpl"
class TestUserInstalledProviderCli:
"""CLI commands of user-installed providers must be discoverable.
Mirror of the relative-import regression above:
discover_plugin_cli_commands() imports the active provider's cli.py as
``_hermes_user_memory.<name>.cli`` without registering the parent
packages, so a cli.py with a relative import could never load.
"""
def _make_plugin_with_cli(self, tmp_path, name):
plugin_dir = tmp_path / "plugins" / name
plugin_dir.mkdir(parents=True)
(plugin_dir / "__init__.py").write_text(
"from agent.memory_provider import MemoryProvider\n"
"from . import config\n"
"class MyProvider(MemoryProvider):\n"
" @property\n"
f" def name(self): return {name!r}\n"
" def is_available(self): return True\n"
" def initialize(self, **kw): pass\n"
" def sync_turn(self, *a, **kw): pass\n"
" def get_tool_schemas(self): return []\n"
" def handle_tool_call(self, *a, **kw): return '{}'\n"
"def register(ctx):\n"
" ctx.register_memory_provider(MyProvider())\n"
)
(plugin_dir / "config.py").write_text("STATUS = 'ok'\n")
(plugin_dir / "cli.py").write_text(
"from . import config\n"
"def register_cli(subparser):\n"
" subparser.add_argument('--status', action='store_true')\n"
)
return plugin_dir
def _activate(self, tmp_path, monkeypatch, name):
monkeypatch.setattr(
"plugins.memory._get_user_plugins_dir",
lambda: tmp_path / "plugins",
)
monkeypatch.setattr(
"plugins.memory._get_active_memory_provider",
lambda: name,
)
def test_cli_discovered_for_user_plugin_with_relative_import(
self, tmp_path, monkeypatch
):
"""discover_plugin_cli_commands() loads a user provider's cli.py."""
from plugins.memory import discover_plugin_cli_commands
self._make_plugin_with_cli(tmp_path, "extcli")
self._activate(tmp_path, monkeypatch, "extcli")
commands = discover_plugin_cli_commands()
assert len(commands) == 1
assert commands[0]["name"] == "extcli"
assert callable(commands[0]["setup_fn"])
def test_provider_load_after_cli_discovery(self, tmp_path, monkeypatch):
"""The provider still loads after CLI discovery ran first.
CLI discovery registers a synthetic parent package shell for the
relative imports in cli.py; _load_provider_from_dir() must load the
real plugin module instead of reusing that shell.
"""
from plugins.memory import discover_plugin_cli_commands, load_memory_provider
self._make_plugin_with_cli(tmp_path, "extcliload")
self._activate(tmp_path, monkeypatch, "extcliload")
assert len(discover_plugin_cli_commands()) == 1
p = load_memory_provider("extcliload")
assert p is not None
assert p.name == "extcliload"
# ---------------------------------------------------------------------------
# Sequential dispatch routing tests
@ -1060,3 +1233,191 @@ class TestHonchoCadenceTracking:
p.on_turn_start(2, "second message")
should_skip = p._injection_frequency == "first-turn" and p._turn_count > 1
assert should_skip, "Second turn (turn 2) SHOULD be skipped"
class TestMemoryToolToolsetGate:
"""Issue #5544: memory provider tools must respect platform_toolsets.
Before the fix, MemoryManager.get_all_tool_schemas() output was appended
to AIAgent.tools unconditionally in agent_init.py — bypassing the
enabled_toolsets filter. Result: `platform_toolsets: telegram: []`
still leaked fact_store and other memory tools into the tool surface,
causing 10x latency on local models (Qwen3-30B: 1.7s → 42s) and
tool-call loops on small models.
These tests mirror the gate logic in agent/agent_init.py around the
memory provider tool injection block. The gate condition is:
enabled_toolsets is None → no filter, inject (backward compat)
"memory" in enabled_toolsets → user opted in, inject
otherwise (incl. []) → skip injection
"""
@staticmethod
def _run_memory_injection(enabled_toolsets, memory_manager):
"""Simulate the gated memory-tool injection block from agent_init.py."""
tools = []
valid_tool_names = set()
if memory_manager and tools is not None and (
enabled_toolsets is None or "memory" in enabled_toolsets
):
_existing = {
t.get("function", {}).get("name")
for t in tools
if isinstance(t, dict)
}
for _schema in memory_manager.get_all_tool_schemas():
_tname = _schema.get("name", "")
if _tname and _tname in _existing:
continue
tools.append({"type": "function", "function": _schema})
if _tname:
valid_tool_names.add(_tname)
_existing.add(_tname)
return tools, valid_tool_names
def _mgr_with_tools(self, *tool_names):
"""Build a MemoryManager whose providers expose the named tool schemas."""
mgr = MemoryManager()
p = FakeMemoryProvider(
"ext",
tools=[{"name": n, "description": n, "parameters": {}} for n in tool_names],
)
mgr.add_provider(p)
return mgr
def test_none_toolsets_injects(self):
"""enabled_toolsets=None (no filter) injects memory tools — backward compat."""
mgr = self._mgr_with_tools("fact_store")
tools, names = self._run_memory_injection(None, mgr)
assert "fact_store" in names
assert any(t["function"]["name"] == "fact_store" for t in tools)
def test_memory_in_toolsets_injects(self):
"""enabled_toolsets including 'memory' injects memory tools."""
mgr = self._mgr_with_tools("fact_store")
tools, names = self._run_memory_injection(["terminal", "memory", "web"], mgr)
assert "fact_store" in names
def test_empty_toolsets_blocks_injection(self):
"""`platform_toolsets: telegram: []` must suppress memory tools. (#5544)"""
mgr = self._mgr_with_tools("fact_store")
tools, names = self._run_memory_injection([], mgr)
assert tools == []
assert names == set()
def test_toolsets_without_memory_blocks_injection(self):
"""Toolset list that doesn't name 'memory' must suppress injection."""
mgr = self._mgr_with_tools("fact_store")
tools, names = self._run_memory_injection(["terminal", "web"], mgr)
assert tools == []
assert names == set()
def test_no_memory_manager_no_injection(self):
"""Gate is moot without a memory manager."""
tools, names = self._run_memory_injection(None, None)
assert tools == []
def test_multiple_schemas_all_blocked_together(self):
"""When the gate is closed, no memory tools leak — not even partially."""
mgr = self._mgr_with_tools("fact_store", "memory_search", "memory_add")
tools, names = self._run_memory_injection(["terminal"], mgr)
assert tools == []
assert names == set()
def test_multiple_schemas_all_injected_when_enabled(self):
"""When the gate is open, every memory tool schema is injected."""
mgr = self._mgr_with_tools("fact_store", "memory_search", "memory_add")
tools, names = self._run_memory_injection(None, mgr)
assert names == {"fact_store", "memory_search", "memory_add"}
class TestContextEngineToolsetGate:
"""Issue #5544 (sibling): context engine tools follow the same gate.
`agent.context_compressor.get_tool_schemas()` (e.g. lcm_grep, lcm_describe,
lcm_expand) was appended to AIAgent.tools unconditionally. Same blind
injection class as the memory bug; same local-model penalty. Gate name:
"context_engine" (matches the existing plugin-system convention).
"""
@staticmethod
def _run_context_engine_injection(enabled_toolsets, compressor):
"""Simulate the gated context-engine injection block from agent_init.py."""
tools = []
valid_tool_names = set()
engine_tool_names = set()
if (
compressor is not None
and tools is not None
and (
enabled_toolsets is None
or "context_engine" in enabled_toolsets
)
):
_existing = {
t.get("function", {}).get("name")
for t in tools
if isinstance(t, dict)
}
for _schema in compressor.get_tool_schemas():
_tname = _schema.get("name", "")
if _tname and _tname in _existing:
continue
tools.append({"type": "function", "function": _schema})
if _tname:
valid_tool_names.add(_tname)
engine_tool_names.add(_tname)
_existing.add(_tname)
return tools, valid_tool_names, engine_tool_names
class _FakeCompressor:
def __init__(self, schemas):
self._schemas = schemas
def get_tool_schemas(self):
return list(self._schemas)
def _compressor_with(self, *tool_names):
return self._FakeCompressor(
[{"name": n, "description": n, "parameters": {}} for n in tool_names]
)
def test_none_toolsets_injects(self):
"""enabled_toolsets=None injects context-engine tools — backward compat."""
c = self._compressor_with("lcm_grep", "lcm_describe", "lcm_expand")
tools, names, engine_names = self._run_context_engine_injection(None, c)
assert engine_names == {"lcm_grep", "lcm_describe", "lcm_expand"}
def test_context_engine_in_toolsets_injects(self):
"""enabled_toolsets including 'context_engine' injects the tools."""
c = self._compressor_with("lcm_grep")
tools, names, engine_names = self._run_context_engine_injection(
["terminal", "context_engine"], c
)
assert "lcm_grep" in engine_names
def test_empty_toolsets_blocks_injection(self):
"""`platform_toolsets: telegram: []` must suppress context-engine tools."""
c = self._compressor_with("lcm_grep")
tools, names, engine_names = self._run_context_engine_injection([], c)
assert tools == []
assert engine_names == set()
def test_toolsets_without_context_engine_blocks_injection(self):
"""A toolset list that doesn't name 'context_engine' suppresses injection."""
c = self._compressor_with("lcm_grep", "lcm_describe")
tools, names, engine_names = self._run_context_engine_injection(
["terminal", "memory"], c
)
assert tools == []
assert engine_names == set()
def test_no_compressor_no_injection(self):
"""Gate is moot without a context_compressor."""
tools, names, engine_names = self._run_context_engine_injection(None, None)
assert tools == []

3
tests/agent/test_memory_session_switch.py
View file

@ -7,7 +7,6 @@ state in initialize() (Hindsight, and any plugin that stores session_id
for scoped writes) keep writing into the old session's record.
"""
import json
import pytest
@ -180,6 +179,7 @@ def test_sync_all_propagates_session_id_to_providers():
p = _RecordingProvider()
mm.add_provider(p)
mm.sync_all("hello", "world", session_id="sess-42")
mm.flush_pending(timeout=5)
assert p.sync_calls == [
{"user": "hello", "asst": "world", "session_id": "sess-42"}
]
@ -190,6 +190,7 @@ def test_queue_prefetch_all_propagates_session_id_to_providers():
p = _RecordingProvider()
mm.add_provider(p)
mm.queue_prefetch_all("next query", session_id="sess-42")
mm.flush_pending(timeout=5)
assert p.queue_calls == [{"query": "next query", "session_id": "sess-42"}]

1
tests/agent/test_memory_user_id.py
View file

@ -6,7 +6,6 @@ so each gateway user gets their own memory bucket instead of sharing a static on
import json
import os
import pytest
from unittest.mock import MagicMock, patch
from agent.memory_provider import MemoryProvider

103
tests/agent/test_minimax_provider.py
View file

@ -4,8 +4,9 @@ from unittest.mock import patch
class TestMinimaxContextLengths:
"""Verify context length entries match official docs (204,800 for all models).
"""Verify context length entries match official docs.
M2.x series is 204,800; M3 is 1M (max output 512K).
Source: https://platform.minimax.io/docs/api-reference/text-anthropic-api
"""
@ -15,11 +16,80 @@ class TestMinimaxContextLengths:
def test_minimax_models_resolve_via_prefix(self):
from agent.model_metadata import get_model_context_length
# All MiniMax models should resolve to 204,800 via the "minimax" prefix
# M2.x models resolve to 204,800 via the "minimax" catch-all
for model in ("MiniMax-M2.7", "MiniMax-M2.5", "MiniMax-M2.1", "MiniMax-M2"):
ctx = get_model_context_length(model, "")
assert ctx == 204_800, f"{model} expected 204800, got {ctx}"
def test_minimax_m3_resolves_to_1m(self):
from agent.model_metadata import get_model_context_length
# M3 must beat the generic "minimax" catch-all (204,800) and resolve to
# a 1M-class context. The exact value depends on the source: our
# hardcoded catalog says 1,000,000; the OpenRouter catalog reports
# 1,048,576 (1024²). Either is correct — assert "≥ 1M, not 204,800".
for model in ("MiniMax-M3", "minimax/minimax-m3", "minimax-m3"):
ctx = get_model_context_length(model, "")
assert ctx >= 1_000_000, f"{model} expected 1M-class, got {ctx}"
class TestMinimaxM3StaleCacheGuard:
"""Pre-catalog builds resolved M3 via the generic 'minimax' catch-all
(204,800) and persisted it before the 'minimax-m3' (1M) catalog entry
existed. The step-1 cache guard must drop that stale value and re-resolve
to 1M, while leaving correct M2.x entries (204,800) untouched.
"""
def test_suggests_minimax_m3(self):
from agent.model_metadata import _model_name_suggests_minimax_m3
assert _model_name_suggests_minimax_m3("MiniMax-M3")
assert _model_name_suggests_minimax_m3("minimax/minimax-m3")
assert not _model_name_suggests_minimax_m3("MiniMax-M2.7")
assert not _model_name_suggests_minimax_m3("MiniMax-M2.5")
def test_stale_m3_cache_dropped_and_reresolves(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
import importlib
import agent.model_metadata as mm
importlib.reload(mm)
base = "https://api.minimaxi.com/anthropic"
mm.save_context_length("MiniMax-M3", base, 204_800)
ctx = mm.get_model_context_length(
"MiniMax-M3", base_url=base, api_key="", provider="minimax-cn"
)
# Invariant: the stale 204,800 catch-all value must be DROPPED and
# re-resolved to M3's real, larger context. The exact value depends on
# the resolution source (hardcoded catalog = 1,000,000; the models.dev
# registry currently reports 512,000) — both are large-context values
# well above the generic "minimax" catch-all. Assert the contract
# ("> 204,800, stale value gone"), not a brittle literal.
assert ctx > 204_800, f"stale M3 cache not dropped/re-resolved, got {ctx}"
def test_correct_m3_cache_preserved(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
import importlib
import agent.model_metadata as mm
importlib.reload(mm)
base = "https://api.minimaxi.com/anthropic"
mm.save_context_length("MiniMax-M3", base, 1_000_000)
ctx = mm.get_model_context_length(
"MiniMax-M3", base_url=base, api_key="", provider="minimax-cn"
)
assert ctx == 1_000_000
def test_m2_cache_not_clobbered(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
import importlib
import agent.model_metadata as mm
importlib.reload(mm)
base = "https://api.minimaxi.com/anthropic"
# 204,800 is the CORRECT value for M2.x — guard must not touch it.
for slug in ("MiniMax-M2.7", "MiniMax-M2.5", "MiniMax-M2.1"):
mm.save_context_length(slug, base, 204_800)
ctx = mm.get_model_context_length(
slug, base_url=base, api_key="", provider="minimax-cn"
)
assert ctx == 204_800, f"{slug} should stay 204800, got {ctx}"
class TestMinimaxThinkingSupport:
@ -71,14 +141,37 @@ class TestMinimaxThinkingSupport:
class TestMinimaxAuxModel:
"""Verify auxiliary model is standard (not highspeed) — now reads from profiles."""
"""Verify auxiliary model is the current frontier standard (not highspeed).
As of M3's release (2026-06-01) the minimax / minimax-cn provider
profiles advertise ``MiniMax-M3`` as their ``default_aux_model`` (the
same model users see in ``_PROVIDER_MODELS["minimax"]`` and in the
user-facing ``model.default`` for a Token-Plan install). The OAuth
/ Coding Plan path sticks with M2.7 because M3 is not on that
tier — see ``test_minimax_profile.py`` for the per-provider split.
The historical concern this class guards is the #4082 / #6082
regression: the highspeed variant costs 2x with no model-quality
benefit, so we still assert that no aux choice contains the substring
``"highspeed"``.
"""
def test_minimax_aux_is_standard(self):
# Import model_tools to trigger plugin discovery so the
# ProviderProfile objects are registered in the providers
# registry before _get_aux_model_for_provider() is called.
# Without this, profile-based resolution can be order-dependent
# or fail outright in isolation (the minimax-* entries are
# no longer in _API_KEY_PROVIDER_AUX_MODELS_FALLBACK after the
# minimax-M3 default-aux-model cleanup, so the profile is
# the only path to a non-empty aux value).
import model_tools # noqa: F401
from agent.auxiliary_client import _get_aux_model_for_provider
assert _get_aux_model_for_provider("minimax") == "MiniMax-M2.7"
assert _get_aux_model_for_provider("minimax-cn") == "MiniMax-M2.7"
assert _get_aux_model_for_provider("minimax") == "MiniMax-M3"
assert _get_aux_model_for_provider("minimax-cn") == "MiniMax-M3"
def test_minimax_aux_not_highspeed(self):
import model_tools # noqa: F401
from agent.auxiliary_client import _get_aux_model_for_provider
assert "highspeed" not in _get_aux_model_for_provider("minimax")
assert "highspeed" not in _get_aux_model_for_provider("minimax-cn")

272
tests/agent/test_model_metadata.py
View file

@ -10,18 +10,15 @@ Coverage levels:
Persistent cache — save/load, corruption, update, provider isolation
"""
import os
import time
import tempfile
import pytest
import yaml
from pathlib import Path
from unittest.mock import patch, MagicMock
from agent.model_metadata import (
CONTEXT_PROBE_TIERS,
DEFAULT_CONTEXT_LENGTHS,
DEFAULT_FALLBACK_CONTEXT,
_strip_provider_prefix,
estimate_tokens_rough,
estimate_messages_tokens_rough,
@ -127,55 +124,6 @@ class TestEstimateMessagesTokensRough:
# =========================================================================
class TestDefaultContextLengths:
def test_claude_models_context_lengths(self):
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
if "claude" not in key:
continue
# Claude 4.6+ models (4.6 and 4.7) have 1M context at standard
# API pricing (no long-context premium). Older Claude 4.x and
# 3.x models cap at 200k.
if any(tag in key for tag in ("4.6", "4-6", "4.7", "4-7")):
assert value == 1000000, f"{key} should be 1000000"
else:
assert value == 200000, f"{key} should be 200000"
def test_gpt4_models_128k_or_1m(self):
# gpt-4.1 and gpt-4.1-mini have 1M context; other gpt-4* have 128k
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
if "gpt-4" in key and "gpt-4.1" not in key:
assert value == 128000, f"{key} should be 128000"
def test_gpt41_models_1m(self):
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
if "gpt-4.1" in key:
assert value == 1047576, f"{key} should be 1047576"
def test_gemini_models_1m(self):
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
if "gemini" in key:
assert value == 1048576, f"{key} should be 1048576"
def test_grok_models_context_lengths(self):
# xAI /v1/models does not return context_length metadata, so
# DEFAULT_CONTEXT_LENGTHS must cover the Grok family explicitly.
# Values sourced from models.dev (2026-04).
expected = {
"grok-4.20": 2000000,
"grok-4-1-fast": 2000000,
"grok-4-fast": 2000000,
"grok-4": 256000,
"grok-code-fast": 256000,
"grok-3": 131072,
"grok-2": 131072,
"grok-2-vision": 8192,
"grok": 131072,
}
for key, value in expected.items():
assert key in DEFAULT_CONTEXT_LENGTHS, f"{key} missing from DEFAULT_CONTEXT_LENGTHS"
assert DEFAULT_CONTEXT_LENGTHS[key] == value, (
f"{key} should be {value}, got {DEFAULT_CONTEXT_LENGTHS[key]}"
)
def test_grok_substring_matching(self):
# Longest-first substring matching must resolve the real xAI model
# IDs to the correct fallback entries without 128k probe-down.
@ -189,12 +137,11 @@ class TestDefaultContextLengths:
("grok-4.20-0309-reasoning", 2000000),
("grok-4.20-0309-non-reasoning", 2000000),
("grok-4.20-multi-agent-0309", 2000000),
("grok-4-1-fast-reasoning", 2000000),
("grok-4-1-fast-non-reasoning", 2000000),
("grok-4-fast-reasoning", 2000000),
("grok-4-fast-non-reasoning", 2000000),
("grok-4", 256000),
("grok-4-0709", 256000),
("grok-build-0.1", 256000),
("grok-code-fast-1", 256000),
("grok-3", 131072),
("grok-3-mini", 131072),
@ -210,6 +157,32 @@ class TestDefaultContextLengths:
f"{model_id}: expected {expected_ctx}, got {actual}"
)
def test_xai_oauth_grok_build_uses_xai_models_dev_context(self):
"""xAI OAuth should share the xAI provider metadata path.
The xAI /v1/models endpoint does not currently include context fields
for grok-build-0.1, so this guards against falling through to the
generic "grok" 131k fallback when using OAuth credentials.
"""
registry = {
"xai": {
"models": {
"grok-build-0.1": {
"limit": {"context": 256000, "output": 64000},
},
},
},
}
with patch("agent.model_metadata.get_cached_context_length", return_value=None), \
patch("agent.model_metadata._query_ollama_api_show", return_value=None), \
patch("agent.models_dev.fetch_models_dev", return_value=registry):
assert get_model_context_length(
"grok-build-0.1",
provider="xai-oauth",
base_url="https://api.x.ai/v1",
api_key="oauth-token",
) == 256000
def test_deepseek_v4_models_1m_context(self):
from agent.model_metadata import get_model_context_length
from unittest.mock import patch as mock_patch
@ -247,12 +220,58 @@ class TestDefaultContextLengths:
f"{model_id}: expected {expected_ctx}, got {actual}"
)
def test_all_values_positive(self):
for key, value in DEFAULT_CONTEXT_LENGTHS.items():
assert value > 0, f"{key} has non-positive context length"
def test_openrouter_live_metadata_beats_hardcoded_catchall(self):
"""OpenRouter-routed slugs resolve via the live OR catalog before the
hardcoded family catch-all.
def test_dict_is_not_empty(self):
assert len(DEFAULT_CONTEXT_LENGTHS) >= 10
Regression for the claude-fable-5 under-report: a brand-new Anthropic
slug that is absent from models.dev but present in OpenRouter's live
catalog (with a 1M window) used to fall through to the generic
``"claude": 200000`` entry, because the step-6 OR fallback was gated on
``not effective_provider`` and ``effective_provider`` is "openrouter"
for any OpenRouter selection. The dedicated step-5 OR branch must read
the live value instead.
"""
from agent.model_metadata import get_model_context_length
from unittest.mock import patch as mock_patch
or_url = "https://openrouter.ai/api/v1"
live = {
"anthropic/claude-fable-5": {"context_length": 1_000_000},
"anthropic/claude-haiku-4.5": {"context_length": 200_000},
}
with mock_patch("agent.model_metadata.fetch_model_metadata", return_value=live), \
mock_patch("agent.model_metadata._query_ollama_api_show", return_value=None), \
mock_patch("agent.model_metadata.get_cached_context_length", return_value=None), \
mock_patch("agent.models_dev.lookup_models_dev_context", return_value=None):
# The bug: would have returned 200_000 via the "claude" catch-all.
assert get_model_context_length(
"anthropic/claude-fable-5", base_url=or_url, provider="openrouter"
) == 1_000_000
# A genuinely-200k model still resolves to its real OR value — the
# fix reads per-model context, it does not blanket-bump to 1M.
assert get_model_context_length(
"anthropic/claude-haiku-4.5", base_url=or_url, provider="openrouter"
) == 200_000
def test_openrouter_kimi_32k_underreport_still_guarded(self):
"""The live OR branch keeps the Kimi-family 32k underreport guard:
a bogus 32768 from OpenRouter for a Kimi slug must NOT win — it falls
through to the hardcoded default instead.
"""
from agent.model_metadata import get_model_context_length
from unittest.mock import patch as mock_patch
or_url = "https://openrouter.ai/api/v1"
live = {"moonshotai/kimi-k2.6": {"context_length": 32768}}
with mock_patch("agent.model_metadata.fetch_model_metadata", return_value=live), \
mock_patch("agent.model_metadata._query_ollama_api_show", return_value=None), \
mock_patch("agent.model_metadata.get_cached_context_length", return_value=None), \
mock_patch("agent.models_dev.lookup_models_dev_context", return_value=None):
ctx = get_model_context_length(
"moonshotai/kimi-k2.6", base_url=or_url, provider="openrouter"
)
assert ctx != 32768, "Kimi 32k OR underreport must not be accepted"
# =========================================================================
@ -808,17 +827,24 @@ class TestGetModelContextLength:
@patch("agent.model_metadata.fetch_model_metadata")
@patch("agent.model_metadata.fetch_endpoint_model_metadata")
def test_custom_endpoint_without_metadata_skips_name_based_default(self, mock_endpoint_fetch, mock_fetch):
def test_custom_endpoint_without_metadata_falls_back_to_catalog(self, mock_endpoint_fetch, mock_fetch):
"""Custom endpoint with no metadata should fall back to the hardcoded
catalog (not 256K) when the model name matches a known entry.
Previously this returned CONTEXT_PROBE_TIERS[0] (256K) because the
custom-endpoint branch short-circuited before the catalog lookup.
See #38865.
"""
mock_fetch.return_value = {}
mock_endpoint_fetch.return_value = {}
# GLM-5-TEE matches the "glm" entry in DEFAULT_CONTEXT_LENGTHS
result = get_model_context_length(
"zai-org/GLM-5-TEE",
base_url="https://llm.chutes.ai/v1",
api_key="test-key",
)
assert result == CONTEXT_PROBE_TIERS[0]
assert result == 202752 # "glm" entry in DEFAULT_CONTEXT_LENGTHS
@patch("agent.model_metadata.fetch_model_metadata")
@patch("agent.model_metadata.fetch_endpoint_model_metadata")
@ -893,6 +919,64 @@ class TestGetModelContextLength:
assert result == 200000
@patch("agent.model_metadata.fetch_model_metadata")
def test_custom_endpoint_falls_back_to_hardcoded_catalog(self, mock_fetch):
"""Custom/proxied endpoint that fails all probes should still resolve
via DEFAULT_CONTEXT_LENGTHS instead of returning 256K.
Regression test for #38865: a corporate Anthropic proxy (custom
base_url) caused the custom-endpoint branch to short-circuit before
the catalog lookup, capping context at 256K even for models like
claude-opus-4-8 that are in the hardcoded catalog with 1M.
"""
mock_fetch.return_value = {}
# Patch all the probe functions that the custom-endpoint branch calls
# so they all fail (return None/empty), simulating a proxy that
# doesn't expose Ollama or local-server endpoints.
with (
patch(
"agent.model_metadata._resolve_endpoint_context_length",
return_value=None,
),
patch(
"agent.model_metadata._query_ollama_api_show",
return_value=None,
),
patch(
"agent.model_metadata._query_local_context_length",
return_value=None,
),
patch(
"agent.model_metadata.is_local_endpoint",
return_value=False,
),
):
# A known model behind a custom proxy should resolve to its
# catalog value (1M), NOT the 256K fallback.
ctx = get_model_context_length(
"claude-opus-4-8",
base_url="https://my-gateway.example.com/v1/claude",
)
assert ctx == 1000000, f"Expected 1000000, got {ctx}"
# Another known model
ctx2 = get_model_context_length(
"claude-sonnet-4-6",
base_url="https://my-gateway.example.com/v1/claude",
)
assert ctx2 == 1000000, f"Expected 1000000, got {ctx2}"
# An unknown model on a custom endpoint should still fall back
# to 256K (no catalog match).
ctx3 = get_model_context_length(
"totally-unknown-model",
base_url="https://my-gateway.example.com/v1/claude",
)
assert ctx3 == DEFAULT_FALLBACK_CONTEXT, (
f"Expected {DEFAULT_FALLBACK_CONTEXT}, got {ctx3}"
)
# =========================================================================
# Bedrock context resolution — must run BEFORE custom-endpoint probe
@ -1120,12 +1204,6 @@ class TestContextProbeTiers:
for i in range(len(CONTEXT_PROBE_TIERS) - 1):
assert CONTEXT_PROBE_TIERS[i] > CONTEXT_PROBE_TIERS[i + 1]
def test_first_tier_is_256k(self):
assert CONTEXT_PROBE_TIERS[0] == 256_000
def test_last_tier_is_8k(self):
assert CONTEXT_PROBE_TIERS[-1] == 8_000
class TestGetNextProbeTier:
def test_from_256k(self):
@ -1288,3 +1366,59 @@ class TestContextLengthCache:
with patch("agent.model_metadata._get_context_cache_path", return_value=cache_file):
save_context_length(model, url, 200000)
assert get_cached_context_length(model, url) == 200000
class TestGrok43StaleCacheGuard:
"""Pre-catalog builds resolved grok-4.3 via the generic 'grok-4' catch-all
(256,000) and persisted it before the 'grok-4.3' (1M) catalog entry was
added on 2026-05-15. The step-1 cache guard must drop that stale value
and re-resolve to 1M, while leaving correct grok-4 entries (256,000)
untouched.
"""
def test_suggests_grok_4_3(self):
from agent.model_metadata import _model_name_suggests_grok_4_3
assert _model_name_suggests_grok_4_3("grok-4.3")
assert _model_name_suggests_grok_4_3("grok-4.3-latest")
assert _model_name_suggests_grok_4_3("xai/grok-4.3")
assert not _model_name_suggests_grok_4_3("grok-4")
assert not _model_name_suggests_grok_4_3("grok-4-fast")
assert not _model_name_suggests_grok_4_3("grok-4.20")
def test_stale_grok_4_3_dropped_and_reresolves_to_1m(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
import importlib
import agent.model_metadata as mm
importlib.reload(mm)
base = "https://api.x.ai/v1"
mm.save_context_length("grok-4.3", base, 256_000)
ctx = mm.get_model_context_length(
"grok-4.3", base_url=base, api_key="", provider="xai"
)
assert ctx == 1_000_000
def test_correct_grok_4_3_cache_preserved(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
import importlib
import agent.model_metadata as mm
importlib.reload(mm)
base = "https://api.x.ai/v1"
mm.save_context_length("grok-4.3", base, 1_000_000)
ctx = mm.get_model_context_length(
"grok-4.3", base_url=base, api_key="", provider="xai"
)
assert ctx == 1_000_000
def test_grok_4_not_clobbered(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
import importlib
import agent.model_metadata as mm
importlib.reload(mm)
base = "https://api.x.ai/v1"
# 256,000 is the CORRECT value for plain grok-4 — guard must not touch it.
for slug in ("grok-4", "grok-4-0709"):
mm.save_context_length(slug, base, 256_000)
ctx = mm.get_model_context_length(
slug, base_url=base, api_key="", provider="xai"
)
assert ctx == 256_000, f"{slug} should stay 256000, got {ctx}"

4
tests/agent/test_model_metadata_local_ctx.py
View file

@ -6,12 +6,10 @@ All tests use synthetic inputs — no filesystem or live server required.
import sys
import os
import json
from unittest.mock import MagicMock, patch
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
import pytest
# ---------------------------------------------------------------------------
@ -562,7 +560,7 @@ class TestGetModelContextLengthLocalFallback:
def test_non_local_endpoint_does_not_query_local_server(self):
"""For non-local endpoints, _query_local_context_length is not called."""
from agent.model_metadata import get_model_context_length, CONTEXT_PROBE_TIERS
from agent.model_metadata import get_model_context_length
with patch("agent.model_metadata.get_cached_context_length", return_value=None), \
patch("agent.model_metadata.fetch_endpoint_model_metadata", return_value={}), \

32
tests/agent/test_models_dev.py
View file

@ -1,8 +1,6 @@
"""Tests for agent.models_dev — models.dev registry integration."""
import json
from unittest.mock import patch, MagicMock
import pytest
from agent.models_dev import (
PROVIDER_TO_MODELS_DEV,
_extract_context,
@ -41,6 +39,16 @@ SAMPLE_REGISTRY = {
},
},
},
"xai": {
"id": "xai",
"name": "xAI",
"models": {
"grok-build-0.1": {
"id": "grok-build-0.1",
"limit": {"context": 256000, "output": 64000},
},
},
},
"kilo": {
"id": "kilo",
"name": "Kilo Gateway",
@ -74,17 +82,9 @@ SAMPLE_REGISTRY = {
class TestProviderMapping:
def test_all_mapped_providers_are_strings(self):
for hermes_id, mdev_id in PROVIDER_TO_MODELS_DEV.items():
assert isinstance(hermes_id, str)
assert isinstance(mdev_id, str)
def test_known_providers_mapped(self):
assert PROVIDER_TO_MODELS_DEV["anthropic"] == "anthropic"
assert PROVIDER_TO_MODELS_DEV["copilot"] == "github-copilot"
assert PROVIDER_TO_MODELS_DEV["stepfun"] == "stepfun"
assert PROVIDER_TO_MODELS_DEV["kilocode"] == "kilo"
assert PROVIDER_TO_MODELS_DEV["ai-gateway"] == "vercel"
def test_xai_oauth_uses_xai_catalog(self):
assert PROVIDER_TO_MODELS_DEV["xai"] == "xai"
assert PROVIDER_TO_MODELS_DEV["xai-oauth"] == "xai"
def test_unmapped_provider_not_in_dict(self):
assert "nous" not in PROVIDER_TO_MODELS_DEV
@ -144,6 +144,12 @@ class TestLookupModelsDevContext:
# GitHub Copilot: only 128K for same model
assert lookup_models_dev_context("copilot", "claude-opus-4.6") == 128000
@patch("agent.models_dev.fetch_models_dev")
def test_xai_oauth_resolves_xai_context(self, mock_fetch):
"""xAI OAuth is an auth path, not a separate model catalog."""
mock_fetch.return_value = SAMPLE_REGISTRY
assert lookup_models_dev_context("xai-oauth", "grok-build-0.1") == 256000
@patch("agent.models_dev.fetch_models_dev")
def test_zero_context_filtered(self, mock_fetch):
mock_fetch.return_value = SAMPLE_REGISTRY

163
tests/agent/test_moonshot_schema.py
View file

@ -6,11 +6,6 @@ the JSON Schema ecosystem accepts:
1. Properties without ``type`` — Moonshot requires ``type`` on every node.
2. ``type`` at the parent of ``anyOf`` — Moonshot requires it only inside
``anyOf`` children.
3. ``$ref`` with sibling keywords — Moonshot expands the ref first and then
rejects ``description``/``type`` siblings on the same node.
(Ported from anomalyco/opencode#24730.)
4. Tuple-style ``items`` arrays — Moonshot requires a single item schema,
not positional ones. (Ported from anomalyco/opencode#24730.)
These tests cover the repairs applied by ``agent/moonshot_schema.py``.
"""
@ -185,164 +180,6 @@ class TestAnyOfParentType:
assert db_type["enum"] == ["mysql", "postgresql"] # "" stripped by enum cleanup
class TestRefSiblingStripping:
"""Rule 4: ``$ref`` nodes may not carry sibling keywords on Moonshot.
Ported from anomalyco/opencode#24730. The real-world failure was MCP tools
whose generated schemas put a ``description`` on a ``$ref`` property so the
model would see the field's human-readable hint. The reference stays — the
referenced definition still owns the description (on the target node itself)
and still serves the model's context.
"""
def test_description_sibling_stripped_from_ref(self):
params = {
"type": "object",
"properties": {
"variantOptions": {
"$ref": "#/$defs/VariantOptions",
"description": "Required. The variant options for generation.",
},
},
"$defs": {
"VariantOptions": {
"type": "object",
"properties": {},
"description": "Configuration options.",
},
},
}
out = sanitize_moonshot_tool_parameters(params)
# Sibling stripped.
assert out["properties"]["variantOptions"] == {"$ref": "#/$defs/VariantOptions"}
# The target definition's own description is preserved — we only strip
# siblings ON the $ref node, not on the thing it points at.
assert out["$defs"]["VariantOptions"]["description"] == "Configuration options."
def test_multiple_siblings_all_stripped(self):
params = {
"type": "object",
"properties": {
"p": {
"$ref": "#/$defs/T",
"type": "object",
"description": "x",
"default": {},
"title": "P",
},
},
"$defs": {"T": {"type": "object"}},
}
out = sanitize_moonshot_tool_parameters(params)
assert out["properties"]["p"] == {"$ref": "#/$defs/T"}
def test_ref_without_siblings_unchanged(self):
params = {
"type": "object",
"properties": {"p": {"$ref": "#/$defs/T"}},
"$defs": {"T": {"type": "object"}},
}
out = sanitize_moonshot_tool_parameters(params)
assert out["properties"]["p"] == {"$ref": "#/$defs/T"}
def test_ref_inside_anyof_children(self):
params = {
"type": "object",
"properties": {
"v": {
"anyOf": [
{"$ref": "#/$defs/A", "description": "variant A"},
{"type": "null"},
],
},
},
"$defs": {"A": {"type": "object"}},
}
out = sanitize_moonshot_tool_parameters(params)
# Main's existing Rule 2 collapses anyOf-with-null down to the
# single non-null branch (Moonshot rejects null branches in anyOf
# outright). That branch was originally `{"$ref": ..., "description": ...}`;
# Rule 4 then strips the sibling, leaving exactly `{"$ref": "..."}`.
# The test name still applies — Rule 4 ran on the $ref branch — it
# just happens after the anyOf collapse on this input.
assert out["properties"]["v"] == {"$ref": "#/$defs/A"}
class TestTupleItems:
"""Rule 5: tuple-style ``items`` arrays collapse to a single schema.
Ported from anomalyco/opencode#24730. Moonshot's schema engine requires
``items`` to be ONE schema object applied to every array element; tuple-
style positional item schemas are rejected. We collapse to the first
element's schema (which is the "closest" interpretation of positional →
single) and drop the rest.
"""
def test_tuple_items_collapsed_to_first(self):
params = {
"type": "object",
"properties": {
"renderedSize": {
"type": "array",
"items": [{"type": "number"}, {"type": "number"}],
"minItems": 2,
"maxItems": 2,
},
},
}
out = sanitize_moonshot_tool_parameters(params)
assert out["properties"]["renderedSize"]["items"] == {"type": "number"}
# Sibling constraints are preserved — only the tuple shape is repaired.
assert out["properties"]["renderedSize"]["minItems"] == 2
def test_empty_tuple_items_becomes_empty_schema(self):
# Empty tuple collapses to ``{}``; the generic repair then fills a
# synthetic ``type`` because Moonshot requires ``type`` on every
# schema node. Either ``{}`` or ``{"type": "string"}`` is a valid
# final shape for Moonshot — both accept any string element — but we
# always go through ``_fill_missing_type`` so the result is fully
# well-formed without needing the consumer to patch it later.
params = {
"type": "object",
"properties": {
"things": {"type": "array", "items": []},
},
}
out = sanitize_moonshot_tool_parameters(params)
items = out["properties"]["things"]["items"]
# Must be a dict and must carry a ``type`` (the whole point of Rule 1).
assert isinstance(items, dict)
assert items.get("type")
def test_tuple_items_first_element_is_repaired(self):
# The first element itself has a missing type — it should be filled.
params = {
"type": "object",
"properties": {
"pair": {
"type": "array",
"items": [{"description": "first"}, {"description": "second"}],
},
},
}
out = sanitize_moonshot_tool_parameters(params)
# Repaired to a single schema with a synthetic type.
assert out["properties"]["pair"]["items"] == {
"description": "first",
"type": "string",
}
def test_single_schema_items_unchanged(self):
params = {
"type": "object",
"properties": {
"tags": {"type": "array", "items": {"type": "string"}},
},
}
out = sanitize_moonshot_tool_parameters(params)
assert out["properties"]["tags"]["items"] == {"type": "string"}
class TestTopLevelGuarantees:
"""The returned top-level schema is always a well-formed object."""

190
tests/agent/test_non_stream_stale_timeout.py Normal file
View file

@ -0,0 +1,190 @@
"""Tests for the non-stream stale-call detector context estimator.
Covers:
- ``estimate_request_context_tokens`` for Chat Completions, Responses API,
bare lists, and mixed-shape dicts.
- ``AIAgent._compute_non_stream_stale_timeout`` with both legacy ``messages``
list and full ``api_kwargs`` dicts.
- The May 2026 default-base change (300s -> 90s) and the lowered
context-tier ceilings (450/600 -> 150/240).
"""
from __future__ import annotations
from pathlib import Path
def _write_config(tmp_path: Path, body: str) -> None:
hermes_home = tmp_path
(hermes_home / "config.yaml").write_text(body or "{}\n", encoding="utf-8")
def _make_agent(tmp_path: Path, **overrides):
from run_agent import AIAgent
kwargs = dict(
model="gpt-5.5",
provider="openai-codex",
api_key="sk-dummy",
base_url="https://chatgpt.com/backend-api/codex",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
platform="cli",
)
kwargs.update(overrides)
return AIAgent(**kwargs)
# ── estimator ──────────────────────────────────────────────────────────────
def test_estimator_chat_completions_messages():
from agent.chat_completion_helpers import estimate_request_context_tokens
payload = {
"model": "gpt-5.4",
"messages": [
{"role": "user", "content": "x" * 400},
{"role": "assistant", "content": "y" * 400},
],
}
# 800+ chars from messages -> ~200 tokens (char/4 estimate)
assert estimate_request_context_tokens(payload) >= 200
def test_estimator_responses_api_input():
from agent.chat_completion_helpers import estimate_request_context_tokens
payload = {
"model": "gpt-5.5",
"instructions": "i" * 1000,
"input": "x" * 4000,
"tools": [{"name": "t", "description": "d" * 200}],
}
# input(4000) + instructions(1000) + tools (~stringified) -> well over 1000 tokens
tokens = estimate_request_context_tokens(payload)
assert tokens >= 1200, f"Responses API estimator returned {tokens}"
def test_estimator_responses_api_long_session_triggers_tier():
"""A real long Codex session (large ``input``) should clear the 50k boundary."""
from agent.chat_completion_helpers import estimate_request_context_tokens
payload = {
"model": "gpt-5.5",
"input": "x" * 240_000, # ~60k tokens (240k chars / 4)
"instructions": "s" * 4000,
}
assert estimate_request_context_tokens(payload) > 50_000
def test_estimator_bare_list_back_compat():
from agent.chat_completion_helpers import estimate_request_context_tokens
messages = [
{"role": "user", "content": "x" * 800},
]
assert estimate_request_context_tokens(messages) >= 200
def test_estimator_empty_inputs():
from agent.chat_completion_helpers import estimate_request_context_tokens
assert estimate_request_context_tokens({}) == 0
assert estimate_request_context_tokens([]) == 0
assert estimate_request_context_tokens(None) == 0
def test_estimator_unknown_dict_fallback():
from agent.chat_completion_helpers import estimate_request_context_tokens
payload = {"random_field": "z" * 400}
assert estimate_request_context_tokens(payload) > 50
# ── default base + tier scaling ────────────────────────────────────────────
def test_default_base_is_90s(monkeypatch, tmp_path):
"""Default base stale timeout dropped from 300s to 90s (May 2026)."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
_write_config(tmp_path, "")
agent = _make_agent(tmp_path)
base, implicit = agent._resolved_api_call_stale_timeout_base()
assert base == 90.0
assert implicit is True
def test_short_codex_request_uses_base_only(monkeypatch, tmp_path):
"""Codex payload below 50k tokens -> default 90s base."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
_write_config(tmp_path, "")
agent = _make_agent(tmp_path)
payload = {"model": "gpt-5.5", "input": "hi", "instructions": ""}
assert agent._compute_non_stream_stale_timeout(payload) == 90.0
def test_long_codex_request_bumps_to_50k_tier(monkeypatch, tmp_path):
"""Codex payload > 50k tokens -> at least 150s."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
_write_config(tmp_path, "")
agent = _make_agent(tmp_path)
payload = {"model": "gpt-5.5", "input": "x" * 240_000, "instructions": ""}
timeout = agent._compute_non_stream_stale_timeout(payload)
assert timeout >= 150.0
assert timeout < 240.0
def test_very_long_codex_request_bumps_to_100k_tier(monkeypatch, tmp_path):
"""Codex payload > 100k tokens -> at least 240s."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
_write_config(tmp_path, "")
agent = _make_agent(tmp_path)
payload = {"model": "gpt-5.5", "input": "x" * 500_000, "instructions": ""}
assert agent._compute_non_stream_stale_timeout(payload) >= 240.0
def test_chat_completions_long_messages_bumps_tier(monkeypatch, tmp_path):
"""Chat Completions estimator still works for the legacy messages path."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
_write_config(tmp_path, "")
agent = _make_agent(
tmp_path,
provider="openai",
base_url="https://api.openai.com/v1",
model="gpt-5.4",
)
payload = {
"model": "gpt-5.4",
"messages": [{"role": "user", "content": "x" * 240_000}],
}
assert agent._compute_non_stream_stale_timeout(payload) >= 150.0
def test_explicit_user_config_overrides_default(monkeypatch, tmp_path):
"""If the user explicitly sets a stale_timeout, the new defaults don't apply."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / ".env").write_text("", encoding="utf-8")
_write_config(tmp_path, """\
providers:
openai-codex:
stale_timeout_seconds: 1800
""")
monkeypatch.delenv("HERMES_API_CALL_STALE_TIMEOUT", raising=False)
import importlib
from hermes_cli import timeouts as to_mod
importlib.reload(to_mod)
agent = _make_agent(tmp_path)
assert agent._compute_non_stream_stale_timeout({"input": "hi"}) == 1800.0

150
tests/agent/test_nous_credits_gauge.py Normal file
View file

@ -0,0 +1,150 @@
"""Tests for the Nous-credits subscription % gauge in build_nous_credits_snapshot.
Covers the monthly_credits denominator path added when the portal /api/oauth/account
subscription block began carrying `monthly_credits`. Magnitudes-only fallback, clamp,
and the non-finite / rollover guards (surfaced by adversarial review) are all asserted.
"""
from hermes_cli.nous_account import (
NousPortalAccountInfo,
NousPaidServiceAccessInfo,
NousPortalSubscriptionInfo,
_subscription_from_payload,
)
from agent.account_usage import build_nous_credits_snapshot, render_account_usage_lines
def _acct(**kwargs):
kwargs.setdefault("logged_in", True)
kwargs.setdefault("source", "account_api")
kwargs.setdefault("fresh", True)
kwargs.setdefault("portal_base_url", "https://portal.nousresearch.com")
return NousPortalAccountInfo(**kwargs)
def _window(snap):
return snap.windows[0] if (snap and snap.windows) else None
def test_parser_captures_monthly_credits():
sub = _subscription_from_payload({
"plan": "Ultra", "tier": 14, "monthly_charge": 200, "monthly_credits": 220,
"current_period_end": "2026-06-28T05:21:54.000Z",
"credits_remaining": 219.27341839, "rollover_credits": 0,
})
assert sub.monthly_credits == 220
assert abs(sub.credits_remaining - 219.27341839) < 1e-6
def test_parser_monthly_credits_absent_is_none():
sub = _subscription_from_payload({"plan": "Ultra", "credits_remaining": 10.0})
assert sub.monthly_credits is None
def test_gauge_present_with_monthly_credits():
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(
plan="Ultra", monthly_credits=220, credits_remaining=219.27341839,
current_period_end="2026-06-28"),
paid_service_access_info=NousPaidServiceAccessInfo(
subscription_credits_remaining=219.27, total_usable_credits=219.27),
))
w = _window(snap)
assert w is not None and w.label == "Subscription"
assert abs(w.used_percent - (220 - 219.27341839) / 220 * 100) < 1e-9
blob = "\n".join(render_account_usage_lines(snap))
assert "% used" in blob or "% remaining" in blob
assert "of $220.00 left" in blob
def test_gauge_90pct():
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(monthly_credits=220, credits_remaining=22.0),
))
assert abs(_window(snap).used_percent - 90.0) < 1e-9
def test_gauge_debt_clamps_to_100():
snap = build_nous_credits_snapshot(_acct(
paid_service_access=False,
subscription=NousPortalSubscriptionInfo(monthly_credits=220, credits_remaining=-5.0),
paid_service_access_info=NousPaidServiceAccessInfo(subscription_credits_remaining=-5.0),
))
assert _window(snap).used_percent == 100.0
def test_gauge_at_cap_is_zero_used():
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(monthly_credits=220, credits_remaining=220.0),
))
assert _window(snap).used_percent == 0.0
def test_no_monthly_credits_falls_back_to_magnitudes():
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(plan="Ultra", credits_remaining=-0.79),
paid_service_access_info=NousPaidServiceAccessInfo(purchased_credits_remaining=991.96),
))
assert _window(snap) is None
blob = "\n".join(render_account_usage_lines(snap))
assert "%" not in blob
assert "Top-up credits: $991.96" in blob
def test_nan_remaining_no_window_no_nan_string():
"""json.loads parses bare NaN by default; isinstance(nan, float) is True.
The gauge must reject it rather than render '$nan' + a false 100% used."""
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(monthly_credits=220, credits_remaining=float("nan")),
paid_service_access_info=NousPaidServiceAccessInfo(purchased_credits_remaining=5.0),
))
assert _window(snap) is None
assert "$nan" not in "\n".join(render_account_usage_lines(snap)).lower()
def test_inf_cap_no_window():
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(monthly_credits=float("inf"), credits_remaining=10.0),
paid_service_access_info=NousPaidServiceAccessInfo(purchased_credits_remaining=5.0),
))
assert _window(snap) is None
def test_rollover_balance_exceeds_cap_no_window():
"""remaining > cap (rollover spanning the period) makes monthly_credits a
nonsensical denominator → suppress the gauge, keep magnitudes."""
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(monthly_credits=220, credits_remaining=300, rollover_credits=80),
paid_service_access_info=NousPaidServiceAccessInfo(subscription_credits_remaining=300.0),
))
assert _window(snap) is None
assert "of $220.00 left" not in "\n".join(render_account_usage_lines(snap))
def test_bool_monthly_credits_no_window():
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(monthly_credits=True, credits_remaining=1.0),
paid_service_access_info=NousPaidServiceAccessInfo(purchased_credits_remaining=5.0),
))
assert _window(snap) is None
def test_zero_monthly_credits_no_divzero():
snap = build_nous_credits_snapshot(_acct(
paid_service_access=True,
subscription=NousPortalSubscriptionInfo(monthly_credits=0, credits_remaining=0.0),
paid_service_access_info=NousPaidServiceAccessInfo(purchased_credits_remaining=5.0),
))
assert _window(snap) is None
def test_failopen_none_and_logged_out():
assert build_nous_credits_snapshot(None) is None
assert build_nous_credits_snapshot(_acct(logged_in=False)) is None

126
tests/agent/test_nous_credits_snapshot.py Normal file
View file

@ -0,0 +1,126 @@
"""Tests for build_nous_credits_snapshot (L6-A, magnitudes-only)."""
from __future__ import annotations
from agent.account_usage import build_nous_credits_snapshot
from hermes_cli.nous_account import (
NousPaidServiceAccessInfo,
NousPortalAccountInfo,
NousPortalSubscriptionInfo,
)
def _account(**kwargs) -> NousPortalAccountInfo:
kwargs.setdefault("logged_in", True)
kwargs.setdefault("source", "account_api")
kwargs.setdefault("fresh", True)
return NousPortalAccountInfo(**kwargs)
def _all_lines(snapshot) -> list[str]:
return list(snapshot.details)
def test_healthy():
info = _account(
paid_service_access=True,
paid_service_access_info=NousPaidServiceAccessInfo(
subscription_credits_remaining=18.0,
purchased_credits_remaining=12.34,
total_usable_credits=30.34,
),
subscription=NousPortalSubscriptionInfo(
plan="Pro",
current_period_end="2026-07-01",
),
)
snap = build_nous_credits_snapshot(info)
assert snap is not None
assert snap.available is True
assert snap.plan == "Pro"
assert snap.provider == "nous"
assert snap.title == "Nous credits"
blob = "\n".join(_all_lines(snap))
assert "$18.00" in blob
assert "$12.34" in blob
assert "$30.34" in blob
assert "Renews: 2026-07-01" in blob
assert "/billing" in blob
# money-rule: magnitudes-only, never a percentage
assert "%" not in blob
def test_money_rule_no_percent():
info = _account(
paid_service_access=True,
paid_service_access_info=NousPaidServiceAccessInfo(
subscription_credits_remaining=18.0,
purchased_credits_remaining=12.34,
total_usable_credits=30.34,
),
subscription=NousPortalSubscriptionInfo(plan="Pro"),
)
snap = build_nous_credits_snapshot(info)
assert snap is not None
for line in snap.details:
assert "%" not in line
def test_depleted():
info = _account(
paid_service_access=False,
paid_service_access_info=NousPaidServiceAccessInfo(
subscription_credits_remaining=0.0,
purchased_credits_remaining=0.0,
total_usable_credits=0.0,
),
subscription=NousPortalSubscriptionInfo(plan="Pro"),
)
snap = build_nous_credits_snapshot(info)
assert snap is not None
blob = "\n".join(_all_lines(snap))
assert "access depleted" in blob
assert "/billing" in blob
def test_purchased_only():
info = _account(
paid_service_access=True,
paid_service_access_info=NousPaidServiceAccessInfo(
subscription_credits_remaining=None,
purchased_credits_remaining=30.0,
total_usable_credits=30.0,
),
subscription=None,
)
snap = build_nous_credits_snapshot(info)
assert snap is not None
blob = "\n".join(_all_lines(snap))
assert "Subscription credits" not in blob
assert "Top-up credits: $30.00" in blob
assert snap.plan is None
def test_logged_out():
info = _account(
logged_in=False,
paid_service_access=True,
paid_service_access_info=NousPaidServiceAccessInfo(
total_usable_credits=10.0,
),
)
assert build_nous_credits_snapshot(info) is None
def test_none():
assert build_nous_credits_snapshot(None) is None
def test_never_raises_empty():
info = _account(
paid_service_access=True,
paid_service_access_info=None,
subscription=None,
)
# No usable numbers and not depleted -> None, without raising.
assert build_nous_credits_snapshot(info) is None

71
tests/agent/test_nous_oauth_401_guidance.py Normal file
View file

@ -0,0 +1,71 @@
"""Tests for the Nous OAuth 401 actionable-guidance branch in
``agent.conversation_loop.run_conversation``.
Source-inspection style (matches ``test_gemini_fast_fallback.py``): we assert
that the guidance strings exist in the function body so that the user-facing
hint cannot be silently removed by a future refactor.
Regression context: ashh hit a Nous 401 (OAuth token expired / portal said
account out of credits) plus a model slug ``deepseek/deepseek-v4-flash:free``
that's OpenRouter syntax, not a Nous catalog name. The previous guidance
branch only covered ``openai-codex`` and ``xai-oauth``; ``nous`` fell through
to a generic "Your API key was rejected... run hermes setup" message, which is
the wrong advice for a pure-OAuth provider.
"""
from __future__ import annotations
import inspect
from agent import conversation_loop
def test_nous_provider_is_in_oauth_401_set():
"""The provider-set gate that selects OAuth-specific guidance must
include ``nous`` alongside ``openai-codex`` and ``xai-oauth``.
"""
source = inspect.getsource(conversation_loop.run_conversation)
# Be flexible about set element ordering — assert all three are listed
# near each other in the gating expression.
assert "\"openai-codex\"" in source
assert "\"xai-oauth\"" in source
assert "\"nous\"" in source
# And the gate string itself must mention all three so future refactors
# that split nous off into its own gate still get caught.
needle = "_provider in {\"openai-codex\", \"xai-oauth\", \"nous\"}"
assert needle in source, (
"Expected nous to be co-gated with the other OAuth providers in the "
"actionable-401-guidance branch of run_conversation."
)
def test_nous_401_guidance_strings_present():
"""User-facing remediation strings for Nous OAuth 401s must exist."""
source = inspect.getsource(conversation_loop.run_conversation)
# Must tell the user it's an OAuth token problem, NOT an API key problem
# (Nous Portal has no API key path — auth_type=oauth_device_code only).
assert "Nous Portal OAuth token was rejected" in source
# Must give a concrete re-auth command, not a generic "hermes setup".
assert "hermes portal" in source
# Must point at the portal so users can check account/credit status.
assert "portal.nousresearch.com" in source
def test_free_slug_hint_for_nous_provider():
"""When the failing model slug ends with ``:free`` and the provider is
``nous``, the guidance must flag that ``:free`` is OpenRouter syntax and
suggest switching providers via ``/model openrouter:<slug>``.
Without this hint, users re-OAuth successfully and then hit the same 401
on the next message because Nous Portal doesn't carry the OpenRouter
free-tier slug.
"""
source = inspect.getsource(conversation_loop.run_conversation)
assert "endswith(\":free\")" in source
assert "OpenRouter slug" in source
assert "/model openrouter:" in source

74
tests/agent/test_onboarding.py
View file

@ -3,7 +3,6 @@
from __future__ import annotations
import yaml
import pytest
from agent.onboarding import (
BUSY_INPUT_FLAG,
@ -237,3 +236,76 @@ class TestOpenclawResidueSeenFlag:
assert mark_seen(cfg_path, OPENCLAW_RESIDUE_FLAG) is True
loaded = yaml.safe_load(cfg_path.read_text())
assert is_seen(loaded, OPENCLAW_RESIDUE_FLAG) is True
class TestProfileBuildMode:
def test_default_is_ask(self):
from agent.onboarding import profile_build_mode
assert profile_build_mode({}) == "ask"
assert profile_build_mode({"onboarding": {}}) == "ask"
assert profile_build_mode({"onboarding": {"profile_build": "ask"}}) == "ask"
def test_off_disables(self):
from agent.onboarding import profile_build_mode
assert profile_build_mode({"onboarding": {"profile_build": "off"}}) == "off"
assert profile_build_mode({"onboarding": {"profile_build": "OFF"}}) == "off"
def test_unknown_value_falls_back_to_ask(self):
from agent.onboarding import profile_build_mode
assert profile_build_mode({"onboarding": {"profile_build": "banana"}}) == "ask"
def test_non_mapping_config_safe(self):
from agent.onboarding import profile_build_mode
assert profile_build_mode("not a dict") == "ask" # type: ignore[arg-type]
assert profile_build_mode({"onboarding": "nope"}) == "ask"
class TestProfileBuildDirective:
def test_directive_is_opt_in_and_consent_gated(self):
from agent.onboarding import profile_build_directive
d = profile_build_directive()
# Must OFFER, not assume.
assert "OFFER" in d
# Must require consent before external lookups.
assert "consent" in d.lower()
# Must forbid silently reading connected accounts.
assert "silently" in d.lower()
# Must persist via the user-profile memory store.
assert 'target="user"' in d
# Must allow declining.
assert "decline" in d.lower()
def test_directive_mentions_first_message(self):
from agent.onboarding import profile_build_directive
assert "first message ever" in profile_build_directive()
class TestProfileBuildSeenFlag:
def test_flag_round_trips(self, tmp_path):
from agent.onboarding import PROFILE_BUILD_FLAG
cfg_path = tmp_path / "config.yaml"
assert mark_seen(cfg_path, PROFILE_BUILD_FLAG) is True
loaded = yaml.safe_load(cfg_path.read_text())
assert is_seen(loaded, PROFILE_BUILD_FLAG) is True
def test_flag_independent_of_busy_input(self, tmp_path):
from agent.onboarding import PROFILE_BUILD_FLAG
cfg_path = tmp_path / "config.yaml"
mark_seen(cfg_path, BUSY_INPUT_FLAG)
loaded = yaml.safe_load(cfg_path.read_text())
assert is_seen(loaded, PROFILE_BUILD_FLAG) is False
class TestProfileBuildConfigDefault:
def test_default_config_carries_ask(self):
from hermes_cli.config import DEFAULT_CONFIG
assert DEFAULT_CONFIG["onboarding"]["profile_build"] == "ask"

1
tests/agent/test_plugin_llm.py
View file

@ -10,7 +10,6 @@ from __future__ import annotations
import asyncio
import base64
import json
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock

81
tests/agent/test_prompt_builder.py
View file

@ -18,7 +18,6 @@ from agent.prompt_builder import (
build_skills_system_prompt,
build_nous_subscription_prompt,
build_context_files_prompt,
build_environment_hints,
CONTEXT_FILE_MAX_CHARS,
DEFAULT_AGENT_IDENTITY,
TOOL_USE_ENFORCEMENT_GUIDANCE,
@ -441,6 +440,7 @@ class TestBuildNousSubscriptionPrompt:
features={
"web": NousFeatureState("web", "Web tools", True, True, True, True, False, True, "firecrawl"),
"image_gen": NousFeatureState("image_gen", "Image generation", True, True, True, True, False, True, "Nous Subscription"),
"video_gen": NousFeatureState("video_gen", "Video generation", False, False, False, False, False, False, ""),
"tts": NousFeatureState("tts", "OpenAI TTS", True, True, True, True, False, True, "OpenAI TTS"),
"stt": NousFeatureState("stt", "Speech-to-text", True, True, True, True, False, True, "OpenAI Whisper"),
"browser": NousFeatureState("browser", "Browser automation", True, True, True, True, False, True, "Browser Use"),
@ -466,6 +466,7 @@ class TestBuildNousSubscriptionPrompt:
features={
"web": NousFeatureState("web", "Web tools", True, False, False, False, False, True, ""),
"image_gen": NousFeatureState("image_gen", "Image generation", True, False, False, False, False, True, ""),
"video_gen": NousFeatureState("video_gen", "Video generation", False, False, False, False, False, False, ""),
"tts": NousFeatureState("tts", "OpenAI TTS", True, False, False, False, False, True, ""),
"stt": NousFeatureState("stt", "Speech-to-text", True, False, False, False, False, True, ""),
"browser": NousFeatureState("browser", "Browser automation", True, False, False, False, False, True, ""),
@ -945,6 +946,29 @@ class TestEnvironmentHints:
assert "Terminal backend: docker" in result
assert "inside" in result.lower()
def test_build_environment_hints_uses_terminal_cwd_over_launch_dir(self, monkeypatch, tmp_path):
"""THE BUG: gateway/cron set TERMINAL_CWD but the prompt emitted os.getcwd()
(the daemon launch dir). Regression for #24882/#24969/#27383/#29265."""
import agent.prompt_builder as _pb
monkeypatch.setattr(_pb, "is_wsl", lambda: False)
monkeypatch.delenv("TERMINAL_ENV", raising=False)
configured = tmp_path / "workspace"
configured.mkdir()
monkeypatch.setenv("TERMINAL_CWD", str(configured))
monkeypatch.chdir(tmp_path)
_pb._clear_backend_probe_cache()
assert f"Current working directory: {configured}" in _pb.build_environment_hints()
def test_build_environment_hints_falls_back_to_launch_dir(self, monkeypatch, tmp_path):
"""The #19242 local-CLI contract: no TERMINAL_CWD → the launch dir."""
import agent.prompt_builder as _pb
monkeypatch.setattr(_pb, "is_wsl", lambda: False)
monkeypatch.delenv("TERMINAL_ENV", raising=False)
monkeypatch.delenv("TERMINAL_CWD", raising=False)
monkeypatch.chdir(tmp_path)
_pb._clear_backend_probe_cache()
assert f"Current working directory: {tmp_path}" in _pb.build_environment_hints()
def test_build_environment_hints_uses_live_probe_when_available(self, monkeypatch):
"""When the probe succeeds, its output must appear in the hint block."""
import agent.prompt_builder as _pb
@ -961,12 +985,64 @@ class TestEnvironmentHints:
def test_remote_backend_list_covers_known_sandboxes(self):
"""Regression guard: if someone adds a remote backend, they must list it here."""
import agent.prompt_builder as _pb
for backend in ("docker", "singularity", "modal", "daytona", "ssh", "vercel_sandbox"):
for backend in ("docker", "singularity", "modal", "daytona", "ssh"):
assert backend in _pb._REMOTE_TERMINAL_BACKENDS, (
f"{backend!r} must be in _REMOTE_TERMINAL_BACKENDS so its host "
f"info is suppressed in the system prompt"
)
def test_environment_hint_from_env_var_is_appended(self, monkeypatch):
"""HERMES_ENVIRONMENT_HINT lets an embedder describe the runtime env."""
import agent.prompt_builder as _pb
monkeypatch.setattr(_pb, "is_wsl", lambda: False)
monkeypatch.delenv("TERMINAL_ENV", raising=False)
monkeypatch.setenv("HERMES_ENVIRONMENT_HINT", "Running inside an OpenShell sandbox.")
_pb._clear_backend_probe_cache()
result = _pb.build_environment_hints()
assert "Running inside an OpenShell sandbox." in result
# The factual host block must still come first.
assert result.index("Host:") < result.index("OpenShell")
def test_environment_hint_env_var_overrides_config(self, monkeypatch):
"""Env var wins over config.yaml agent.environment_hint."""
import agent.prompt_builder as _pb
monkeypatch.setattr(_pb, "is_wsl", lambda: False)
monkeypatch.delenv("TERMINAL_ENV", raising=False)
monkeypatch.setenv("HERMES_ENVIRONMENT_HINT", "ENV-WINS")
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"agent": {"environment_hint": "CONFIG-VALUE"}},
)
_pb._clear_backend_probe_cache()
result = _pb.build_environment_hints()
assert "ENV-WINS" in result
assert "CONFIG-VALUE" not in result
def test_environment_hint_falls_back_to_config(self, monkeypatch):
"""With no env var, the config.yaml value is used."""
import agent.prompt_builder as _pb
monkeypatch.setattr(_pb, "is_wsl", lambda: False)
monkeypatch.delenv("TERMINAL_ENV", raising=False)
monkeypatch.delenv("HERMES_ENVIRONMENT_HINT", raising=False)
monkeypatch.setattr(
"hermes_cli.config.load_config",
lambda: {"agent": {"environment_hint": "CONFIG-VALUE"}},
)
_pb._clear_backend_probe_cache()
result = _pb.build_environment_hints()
assert "CONFIG-VALUE" in result
def test_environment_hint_empty_by_default(self, monkeypatch):
"""No hint configured anywhere → no embedder text, host block intact."""
import agent.prompt_builder as _pb
monkeypatch.setattr(_pb, "is_wsl", lambda: False)
monkeypatch.delenv("TERMINAL_ENV", raising=False)
monkeypatch.delenv("HERMES_ENVIRONMENT_HINT", raising=False)
monkeypatch.setattr("hermes_cli.config.load_config", lambda: {"agent": {}})
_pb._clear_backend_probe_cache()
result = _pb.build_environment_hints()
assert "Host:" in result
# =========================================================================
# Conditional skill activation
@ -1213,4 +1289,3 @@ class TestOpenAIModelExecutionGuidance:
# =========================================================================

2
tests/agent/test_prompt_caching.py
View file

@ -1,7 +1,5 @@
"""Tests for agent/prompt_caching.py — Anthropic cache control injection."""
import copy
import pytest
from agent.prompt_caching import (
_apply_cache_marker,

3
tests/agent/test_rate_limit_tracker.py
View file

@ -189,14 +189,11 @@ class TestAgentIntegration:
def test_capture_rate_limits_from_headers(self):
"""Simulate the header capture path without a real API call."""
import sys
import os
# Use a mock httpx-like response
class MockResponse:
headers = NOUS_HEADERS
# Import AIAgent minimally
from unittest.mock import MagicMock, patch
# Test the parsing directly
state = parse_rate_limit_headers(MockResponse.headers, provider="nous")

159
tests/agent/test_redact.py
View file

@ -1,7 +1,6 @@
"""Tests for agent.redact -- secret masking in logs and output."""
import logging
import os
import pytest
@ -343,140 +342,86 @@ class TestJWTTokens:
class TestDiscordMentions:
"""Discord snowflake IDs in <@ID> or <@!ID> format."""
"""Discord mention snowflakes (<@ID> / <@!ID>) are public syntax, not
secrets — they must pass through the redactor unchanged so multi-bot
@-pings (DISCORD_ALLOW_BOTS=mentions) keep resolving. See issue #35611."""
def test_normal_mention(self):
result = redact_sensitive_text("Hello <@222589316709220353>")
assert "222589316709220353" not in result
assert "<@***>" in result
def test_normal_mention_passes_through(self):
text = "Hello <@222589316709220353>"
assert redact_sensitive_text(text) == text
def test_nickname_mention(self):
result = redact_sensitive_text("Ping <@!1331549159177846844>")
assert "1331549159177846844" not in result
assert "<@!***>" in result
def test_nickname_mention_passes_through(self):
text = "Ping <@!1331549159177846844>"
assert redact_sensitive_text(text) == text
def test_multiple_mentions(self):
def test_multiple_mentions_pass_through(self):
text = "<@111111111111111111> and <@222222222222222222>"
result = redact_sensitive_text(text)
assert "111111111111111111" not in result
assert "222222222222222222" not in result
assert redact_sensitive_text(text) == text
def test_short_id_not_matched(self):
"""IDs shorter than 17 digits are not Discord snowflakes."""
def test_short_id_passes_through(self):
text = "<@12345>"
assert redact_sensitive_text(text) == text
def test_slack_mention_not_matched(self):
"""Slack mentions use letters, not pure digits."""
def test_slack_mention_passes_through(self):
text = "<@U024BE7LH>"
assert redact_sensitive_text(text) == text
def test_preserves_surrounding_text(self):
text = "User <@222589316709220353> said hello"
result = redact_sensitive_text(text)
assert result.startswith("User ")
assert result.endswith(" said hello")
assert redact_sensitive_text(text) == text
class TestUrlQueryParamRedaction:
"""URL query-string redaction (ported from nearai/ironclaw#2529).
Catches opaque tokens that don't match vendor prefix regexes by
matching on parameter NAME rather than value shape.
class TestWebUrlsNotRedacted:
"""Web URLs (http/https/wss) pass through unchanged — magic-link
checkouts, OAuth callbacks the agent is meant to follow, and pre-signed
share URLs must reach the tool intact. Known credential shapes inside
URLs (sk-, ghp_, JWTs) are still caught by the prefix and JWT regexes.
DB connection-string passwords are still caught by _DB_CONNSTR_RE.
"""
def test_oauth_callback_code(self):
def test_oauth_callback_code_passes_through(self):
text = "GET https://api.example.com/oauth/cb?code=abc123xyz789&state=csrf_ok"
result = redact_sensitive_text(text)
assert "abc123xyz789" not in result
assert "code=***" in result
assert "state=csrf_ok" in result # state is not sensitive
assert redact_sensitive_text(text) == text
def test_access_token_query(self):
def test_access_token_query_passes_through(self):
text = "Fetching https://example.com/api?access_token=opaque_value_here_1234&format=json"
result = redact_sensitive_text(text)
assert "opaque_value_here_1234" not in result
assert "access_token=***" in result
assert "format=json" in result
assert redact_sensitive_text(text) == text
def test_refresh_token_query(self):
text = "https://auth.example.com/token?refresh_token=somerefresh&grant_type=refresh"
result = redact_sensitive_text(text)
assert "somerefresh" not in result
assert "grant_type=refresh" in result
def test_magic_link_checkout_passes_through(self):
text = "Open https://checkout.example.com/resume?magic=ABCDEF123456&customer=42"
assert redact_sensitive_text(text) == text
def test_api_key_query(self):
text = "https://api.example.com/v1/data?api_key=kABCDEF12345&limit=10"
result = redact_sensitive_text(text)
assert "kABCDEF12345" not in result
assert "limit=10" in result
def test_presigned_signature(self):
def test_presigned_signature_passes_through(self):
text = "https://s3.amazonaws.com/bucket/k?signature=LONG_PRESIGNED_SIG&id=public"
result = redact_sensitive_text(text)
assert "LONG_PRESIGNED_SIG" not in result
assert "id=public" in result
def test_case_insensitive_param_names(self):
"""Lowercase/mixed-case sensitive param names are redacted."""
# NOTE: All-caps names like TOKEN= are swallowed by _ENV_ASSIGN_RE
# (which matches KEY=value patterns greedily) before URL regex runs.
# This test uses lowercase names to isolate URL-query redaction.
text = "https://example.com?api_key=abcdef&secret=ghijkl"
result = redact_sensitive_text(text)
assert "abcdef" not in result
assert "ghijkl" not in result
assert "api_key=***" in result
assert "secret=***" in result
def test_substring_match_does_not_trigger(self):
"""`token_count` and `session_id` must NOT match `token` / `session`."""
text = "https://example.com/cb?token_count=42&session_id=xyz&foo=bar"
result = redact_sensitive_text(text)
assert "token_count=42" in result
assert "session_id=xyz" in result
def test_url_without_query_unchanged(self):
text = "https://example.com/path/to/resource"
assert redact_sensitive_text(text) == text
def test_url_with_fragment(self):
text = "https://example.com/page?token=xyz#section"
result = redact_sensitive_text(text)
assert "token=xyz" not in result
assert "#section" in result
def test_websocket_url_query(self):
text = "wss://api.example.com/ws?token=opaqueWsToken123"
result = redact_sensitive_text(text)
assert "opaqueWsToken123" not in result
class TestUrlUserinfoRedaction:
"""URL userinfo (`scheme://user:pass@host`) for non-DB schemes."""
def test_https_userinfo(self):
def test_https_userinfo_passes_through(self):
text = "URL: https://user:supersecretpw@host.example.com/path"
result = redact_sensitive_text(text)
assert "supersecretpw" not in result
assert "https://user:***@host.example.com" in result
def test_http_userinfo(self):
text = "http://admin:plaintextpass@internal.example.com/api"
result = redact_sensitive_text(text)
assert "plaintextpass" not in result
def test_ftp_userinfo(self):
text = "ftp://user:ftppass@ftp.example.com/file.txt"
result = redact_sensitive_text(text)
assert "ftppass" not in result
def test_url_without_userinfo_unchanged(self):
text = "https://example.com/path"
assert redact_sensitive_text(text) == text
def test_db_connstr_still_handled(self):
"""DB schemes are handled by _DB_CONNSTR_RE, not _URL_USERINFO_RE."""
def test_websocket_url_query_passes_through(self):
text = "wss://api.example.com/ws?token=opaqueWsToken123"
assert redact_sensitive_text(text) == text
def test_http_access_log_request_target_passes_through(self):
text = (
'INFO aiohttp.access: 127.0.0.1 "POST '
'/bluebubbles-webhook?password=webhookSecret123&event=new-message '
'HTTP/1.1" 200 173 "-" "test-client"'
)
assert redact_sensitive_text(text) == text
def test_known_prefix_inside_url_still_redacted(self):
"""sk-/ghp_/JWT-shaped values inside a URL are still caught by
_PREFIX_RE / _JWT_RE — the carve-out is for opaque tokens only."""
text = "https://evil.com/steal?key=sk-" + "a" * 30
result = redact_sensitive_text(text)
assert "sk-" + "a" * 30 not in result
def test_db_connstr_password_still_redacted(self):
"""DB schemes (postgres/mysql/mongodb/redis/amqp) keep their
userinfo redaction via _DB_CONNSTR_RE — connection strings are
not web URLs the agent navigates to."""
text = "postgres://admin:dbpass@db.internal:5432/app"
result = redact_sensitive_text(text)
assert "dbpass" not in result

141
tests/agent/test_resume_stale_active_task.py Normal file
View file

@ -0,0 +1,141 @@
"""Regression coverage for #35344: a resumed session must not let a stale
``## Active Task`` from an inherited compaction handoff hijack the reply to a
new, unrelated user message.
The failure mode (real report): a lineage was compacted, producing a handoff
whose ``## Active Task`` described task A. The lineage was resumed later and
the user asked about an unrelated task B. The model answered with A because
the handoff's resume directive outranked the fresh ask.
The structural fix lives in ``SUMMARY_PREFIX``: the handoff is framed as
reference-only and the latest user message explicitly *wins* on conflict, with
named reverse-signal verbs. Two invariants guard the resume path specifically:
1. A handoff persisted under the OLD (conflicting) prefix is re-normalized to
the CURRENT prefix when it is re-compacted on a resumed lineage — so a
pre-fix stale handoff cannot keep its "resume exactly" directive forever.
2. The current handoff prefix contains an unambiguous "latest message wins /
discard stale Active Task" rule, so an unrelated new ask is privileged over
the inherited ``## Active Task``.
These are content/structural assertions (no live model call) — they pin the
mechanism that makes the stale task historical rather than active.
"""
from agent.context_compressor import (
SUMMARY_PREFIX,
LEGACY_SUMMARY_PREFIX,
ContextCompressor,
)
# The conflicting prefix that shipped before the #35344 fix. A handoff
# persisted in a resumed lineage could carry this verbatim.
_OLD_CONFLICTING_PREFIX = (
"[CONTEXT COMPACTION — REFERENCE ONLY] Earlier turns were compacted "
"into the summary below. This is a handoff from a previous context "
"window — treat it as background reference, NOT as active instructions. "
"Do NOT answer questions or fulfill requests mentioned in this summary; "
"they were already addressed. "
"Your current task is identified in the '## Active Task' section of the "
"summary — resume exactly from there. "
"Respond ONLY to the latest user message "
"that appears AFTER this summary. The current session state (files, "
"config, etc.) may reflect work described here — avoid repeating it:"
)
def test_latest_message_wins_over_inherited_active_task():
"""The handoff must explicitly privilege the latest user message over a
stale ``## Active Task`` — the core #35344 contract."""
lower = SUMMARY_PREFIX.lower()
assert "latest user message" in lower
assert "## active task" in lower
# Conflict-resolution must be explicit, not implied.
assert "wins" in lower or "supersede" in lower
assert "discard" in lower
def test_no_resume_exactly_directive_can_hijack():
"""The directive that caused the hijack ("resume exactly from Active
Task") must be gone."""
assert "resume exactly" not in SUMMARY_PREFIX.lower()
def test_resumed_stale_handoff_gets_renormalized_to_current_prefix():
"""A handoff persisted under the OLD conflicting prefix (e.g. saved before
the fix and inherited into a resumed lineage) is upgraded to the CURRENT
prefix when re-normalized on re-compaction — so the "resume exactly"
directive cannot survive into a resumed session."""
stale_body = (
"## Active Task\n"
"User asked: 'Migrate the billing module to Stripe'\n\n"
"## Goal\nMigrate billing.\n"
)
stale_handoff = f"{_OLD_CONFLICTING_PREFIX}\n{stale_body}"
# Sanity: the fixture really does carry the old directive.
assert "resume exactly" in stale_handoff.lower()
renormalized = ContextCompressor._with_summary_prefix(stale_handoff)
# The body is preserved...
assert "Migrate the billing module to Stripe" in renormalized
# ...but the conflicting directive is stripped and replaced with the
# current latest-message-wins framing.
assert "resume exactly" not in renormalized.lower()
assert renormalized.startswith(SUMMARY_PREFIX)
assert "wins" in renormalized.lower()
def test_legacy_prefix_handoff_also_renormalized():
"""The same upgrade applies to the oldest ``[CONTEXT SUMMARY]:`` handoff
format that may sit in a long-lived resumed lineage."""
legacy = f"{LEGACY_SUMMARY_PREFIX} ## Active Task\nUser asked: 'task A'"
renormalized = ContextCompressor._with_summary_prefix(legacy)
assert renormalized.startswith(SUMMARY_PREFIX)
assert LEGACY_SUMMARY_PREFIX not in renormalized
assert "task A" in renormalized
def test_inherited_handoff_detected_in_resumed_protected_head():
"""On a resumed lineage the handoff commonly sits right after the system
prompt (in the protected head). ``_find_latest_context_summary`` must
detect it there so re-compaction rehydrates state from it rather than
serializing it as a fresh user turn (which is what let the stale Active
Task read as live intent)."""
messages = [
{"role": "system", "content": "system prompt"},
{"role": "user", "content": f"{SUMMARY_PREFIX}\n## Active Task\nUser asked: 'task A'"},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": "Unrelated task B: what's the capital of France?"},
]
# Search the whole post-system range.
idx, body = ContextCompressor._find_latest_context_summary(
messages, 1, len(messages)
)
assert idx == 1, "handoff in protected head must be found"
assert "task A" in body
# The detected body is stripped of the prefix (treated as state, not a
# standalone instruction message).
assert not body.startswith(SUMMARY_PREFIX)
def test_historical_prefixed_handoff_detected_and_stripped():
"""A pre-fix handoff (old conflicting prefix) inherited into a resumed
lineage must still be recognized as a context summary AND have its old
directive stripped on detection — otherwise re-compaction serializes the
stale 'resume exactly' text as a fresh turn."""
messages = [
{"role": "system", "content": "system prompt"},
{"role": "user", "content": f"{_OLD_CONFLICTING_PREFIX}\n## Active Task\nUser asked: 'task A'"},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": "Unrelated task B"},
]
idx, body = ContextCompressor._find_latest_context_summary(
messages, 1, len(messages)
)
assert idx == 1
assert "task A" in body
assert "resume exactly" not in body.lower()

129
tests/agent/test_runtime_cwd.py Normal file
View file

@ -0,0 +1,129 @@
"""Tests for agent/runtime_cwd.py — the single source of truth for the agent working directory."""
import os
from pathlib import Path
import pytest
import agent.runtime_cwd as rt
from agent.runtime_cwd import (
clear_session_cwd,
resolve_agent_cwd,
resolve_context_cwd,
set_session_cwd,
)
def _raise_oserror(*args, **kwargs):
raise OSError("cwd gone")
class TestResolveAgentCwd:
def test_prefers_terminal_cwd_over_getcwd(self, monkeypatch, tmp_path):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
monkeypatch.chdir(os.path.expanduser("~"))
assert resolve_agent_cwd() == tmp_path
def test_falls_back_to_getcwd_when_unset(self, monkeypatch, tmp_path):
# The #19242 local-CLI contract: TERMINAL_CWD is unset, so the launch dir wins.
monkeypatch.delenv("TERMINAL_CWD", raising=False)
monkeypatch.chdir(tmp_path)
assert resolve_agent_cwd() == tmp_path
def test_skips_nonexistent_terminal_cwd(self, monkeypatch, tmp_path):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path / "gone"))
monkeypatch.chdir(tmp_path)
assert resolve_agent_cwd() == tmp_path
def test_expands_leading_tilde(self, monkeypatch):
monkeypatch.setenv("TERMINAL_CWD", "~")
assert resolve_agent_cwd() == Path(os.path.expanduser("~"))
def test_whitespace_only_terminal_cwd_falls_back_to_getcwd(self, monkeypatch, tmp_path):
# " ".strip() → "" → falsy, so the launch dir wins (not a " " path).
monkeypatch.setenv("TERMINAL_CWD", " ")
monkeypatch.chdir(tmp_path)
assert resolve_agent_cwd() == tmp_path
def test_propagates_oserror_from_getcwd(self, monkeypatch):
# The fallback arm calls os.getcwd(), which can raise OSError (deleted cwd).
# The resolver must NOT swallow it — build_environment_hints owns the
# try/except OSError guard at the call site (prompt_builder.py:805).
monkeypatch.delenv("TERMINAL_CWD", raising=False)
monkeypatch.setattr(rt.os, "getcwd", _raise_oserror)
with pytest.raises(OSError):
resolve_agent_cwd()
class TestResolveContextCwd:
def test_returns_dir_when_set(self, monkeypatch, tmp_path):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
assert resolve_context_cwd() == tmp_path
def test_returns_none_when_unset(self, monkeypatch):
# Unset → None; the caller (build_context_files_prompt) then getcwds —
# the local-CLI #19242 contract. Discovery still runs; it is NOT skipped.
monkeypatch.delenv("TERMINAL_CWD", raising=False)
assert resolve_context_cwd() is None
def test_returns_nonexistent_dir_unguarded(self, monkeypatch, tmp_path):
# Deliberate asymmetry vs resolve_agent_cwd: context discovery has no isdir
# guard, so a missing dir is returned (not None) — discovery just finds nothing.
missing = tmp_path / "gone"
monkeypatch.setenv("TERMINAL_CWD", str(missing))
assert resolve_context_cwd() == missing
def test_expands_leading_tilde(self, monkeypatch):
monkeypatch.setenv("TERMINAL_CWD", "~")
assert resolve_context_cwd() == Path(os.path.expanduser("~"))
def test_whitespace_only_terminal_cwd_returns_none(self, monkeypatch):
# " ".strip() → "" → None, so the caller getcwds for discovery rather
# than building Path(" ") and resolving garbage under the launch dir.
monkeypatch.setenv("TERMINAL_CWD", " ")
assert resolve_context_cwd() is None
class TestSessionCwdOverride:
"""The #29531 per-session arm: a contextvar cwd wins over TERMINAL_CWD so a
multi-session gateway can pin each session to its own folder."""
def test_session_cwd_overrides_terminal_cwd(self, monkeypatch, tmp_path):
other = tmp_path / "other"
other.mkdir()
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
token = set_session_cwd(str(other))
try:
assert resolve_agent_cwd() == other
assert resolve_context_cwd() == other
finally:
rt._SESSION_CWD.reset(token)
def test_empty_session_cwd_falls_back_to_terminal_cwd(self, monkeypatch, tmp_path):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
token = set_session_cwd("")
try:
assert resolve_agent_cwd() == tmp_path
assert resolve_context_cwd() == tmp_path
finally:
rt._SESSION_CWD.reset(token)
def test_clear_session_cwd_restores_terminal_cwd(self, monkeypatch, tmp_path):
other = tmp_path / "other"
other.mkdir()
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
token = set_session_cwd(str(other))
try:
clear_session_cwd()
assert resolve_agent_cwd() == tmp_path
finally:
rt._SESSION_CWD.reset(token)
def test_nonexistent_session_cwd_falls_back(self, monkeypatch, tmp_path):
monkeypatch.setenv("TERMINAL_CWD", str(tmp_path))
token = set_session_cwd(str(tmp_path / "gone"))
try:
# resolve_agent_cwd guards on isdir; a missing session cwd must not win.
assert resolve_agent_cwd() == tmp_path
finally:
rt._SESSION_CWD.reset(token)

168
tests/agent/test_save_url_image.py Normal file
View file

@ -0,0 +1,168 @@
"""Direct tests for ``agent.image_gen_provider.save_url_image`` (#26942).
These exercise the helper against a real in-process HTTP server — no
``requests.get`` mocking — so we catch the kinds of issues a mocked
unit test won't: content-type parsing, partial-write cleanup, the
oversize cap, the empty-body refusal, and the cache directory it
actually writes to.
Pre-fix the helper didn't exist; xAI URL responses were returned bare
and the gateway 404'd at ``send_photo`` time.
"""
from __future__ import annotations
import http.server
import socketserver
import threading
import pytest
PNG_1PX = bytes.fromhex(
"89504e470d0a1a0a0000000d49484452000000010000000108020000009077"
"53de00000010494441547801635c0e000000feff03000006000557bfabd400"
"00000049454e44ae426082"
)
class _TinyImageHandler(http.server.BaseHTTPRequestHandler):
"""Tiny HTTP server that mimics the shapes save_url_image must handle."""
def do_GET(self): # noqa: N802
if self.path == "/image.png":
self.send_response(200)
self.send_header("Content-Type", "image/png")
self.send_header("Content-Length", str(len(PNG_1PX)))
self.end_headers()
self.wfile.write(PNG_1PX)
elif self.path == "/image.jpg":
self.send_response(200)
self.send_header("Content-Type", "image/jpeg")
self.end_headers()
self.wfile.write(PNG_1PX) # bytes don't have to be a real jpeg
elif self.path == "/oversize":
self.send_response(200)
self.send_header("Content-Type", "image/png")
self.end_headers()
chunk = b"\x00" * 65536
for _ in range(64): # 4 MiB
self.wfile.write(chunk)
elif self.path == "/empty":
self.send_response(200)
self.send_header("Content-Type", "image/png")
self.send_header("Content-Length", "0")
self.end_headers()
elif self.path == "/404":
self.send_response(404)
self.end_headers()
elif self.path == "/no-type-with-url-ext.jpg":
self.send_response(200)
self.send_header("Content-Type", "application/octet-stream")
self.end_headers()
self.wfile.write(PNG_1PX)
elif self.path == "/no-type-no-ext":
self.send_response(200)
self.end_headers()
self.wfile.write(PNG_1PX)
else:
self.send_response(404)
self.end_headers()
def log_message(self, *args, **kw): # noqa: D401
return
@pytest.fixture
def http_server(tmp_path, monkeypatch):
"""Spin up a localhost HTTP server and isolate HERMES_HOME under tmp_path."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
(tmp_path / ".hermes").mkdir()
# Force the constants/image cache helpers to re-read HERMES_HOME.
import sys
for mod in list(sys.modules):
if mod.startswith("hermes_constants") or mod.startswith("agent.image_gen_provider"):
sys.modules.pop(mod, None)
httpd = socketserver.TCPServer(("127.0.0.1", 0), _TinyImageHandler)
port = httpd.server_address[1]
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
thread.start()
yield f"http://127.0.0.1:{port}", httpd
httpd.shutdown()
class TestSaveUrlImage:
def test_writes_real_bytes_to_hermes_home_cache(self, http_server):
base, _ = http_server
from agent.image_gen_provider import save_url_image
path = save_url_image(f"{base}/image.png", prefix="xai_test")
assert path.exists()
assert path.read_bytes() == PNG_1PX
# The cache directory must be under HERMES_HOME — gateway cleanup
# relies on this being the canonical location.
assert "cache/images" in str(path)
assert path.suffix == ".png"
def test_extension_inferred_from_content_type(self, http_server):
base, _ = http_server
from agent.image_gen_provider import save_url_image
path = save_url_image(f"{base}/image.jpg", prefix="xai_test")
assert path.suffix == ".jpg", "image/jpeg → .jpg"
def test_extension_falls_back_to_url_suffix(self, http_server):
"""Some CDNs send ``application/octet-stream`` — the URL suffix wins then."""
base, _ = http_server
from agent.image_gen_provider import save_url_image
path = save_url_image(f"{base}/no-type-with-url-ext.jpg", prefix="xai_test")
assert path.suffix == ".jpg"
def test_extension_defaults_to_png_when_unknowable(self, http_server):
base, _ = http_server
from agent.image_gen_provider import save_url_image
path = save_url_image(f"{base}/no-type-no-ext", prefix="xai_test")
assert path.suffix == ".png"
def test_404_raises(self, http_server):
"""HTTP errors must propagate — caller decides whether to fall back."""
base, _ = http_server
from agent.image_gen_provider import save_url_image
import requests as req_lib
with pytest.raises(req_lib.HTTPError):
save_url_image(f"{base}/404")
def test_empty_body_raises_without_writing_file(self, http_server):
"""0-byte responses are not images — refuse to cache."""
base, _ = http_server
from agent.image_gen_provider import save_url_image
with pytest.raises(ValueError, match="0 bytes"):
save_url_image(f"{base}/empty")
def test_oversize_raises_and_cleans_up(self, http_server, tmp_path):
"""Oversize downloads must NOT leak a partial file into the cache."""
base, _ = http_server
from agent.image_gen_provider import save_url_image, _images_cache_dir
cache_dir = _images_cache_dir()
before = set(cache_dir.glob("*"))
with pytest.raises(ValueError, match="exceeds"):
save_url_image(f"{base}/oversize", max_bytes=1024 * 1024)
after = set(cache_dir.glob("*"))
assert after == before, "partial file leaked into cache after oversize cap"
def test_unique_filenames_avoid_collision(self, http_server):
"""Two back-to-back saves of the same URL must produce different paths."""
base, _ = http_server
from agent.image_gen_provider import save_url_image
path1 = save_url_image(f"{base}/image.png", prefix="xai_collision")
path2 = save_url_image(f"{base}/image.png", prefix="xai_collision")
assert path1 != path2, "filename collision — uuid suffix isn't doing its job"

226
tests/agent/test_set_runtime_main_custom_provider.py Normal file
View file

@ -0,0 +1,226 @@
"""Regression test: set_runtime_main() must pass base_url/api_key/api_mode
so that _resolve_auto() can route custom: providers in Step 1.
Fixes https://github.com/NousResearch/hermes-agent/issues/34777
"""
import pytest
from unittest.mock import patch, MagicMock
def _get_globals(mod):
"""Read runtime globals without triggering redaction."""
return {
"provider": mod._RUNTIME_MAIN_PROVIDER,
"model": mod._RUNTIME_MAIN_MODEL,
"base_url": mod._RUNTIME_MAIN_BASE_URL,
"cred": mod._RUNTIME_MAIN_API_KEY, # renamed to avoid redaction
"api_mode": mod._RUNTIME_MAIN_API_MODE,
}
class TestSetRuntimeMainCustomProvider:
"""set_runtime_main must propagate base_url/api_key/api_mode for custom providers."""
def test_globals_stored(self):
"""set_runtime_main stores all five fields in process-local globals."""
import agent.auxiliary_client as mod
mod.clear_runtime_main()
try:
mod.set_runtime_main(
"custom:my-router",
"glm-5.1",
base_url="https://my-server.example.com/v1",
api_key="sk-test-key",
api_mode="chat_completions",
)
g = _get_globals(mod)
assert g["provider"] == "custom:my-router"
assert g["model"] == "glm-5.1"
assert g["base_url"] == "https://my-server.example.com/v1"
assert g["cred"] == "sk-test-key"
assert g["api_mode"] == "chat_completions"
finally:
mod.clear_runtime_main()
def test_clear_resets_all_globals(self):
"""clear_runtime_main resets all five globals to empty."""
import agent.auxiliary_client as mod
mod.set_runtime_main(
"custom:x", "m",
base_url="https://x.example.com",
api_key="sk-abc",
api_mode="chat_completions",
)
mod.clear_runtime_main()
g = _get_globals(mod)
for v in g.values():
assert v == "", f"Expected empty, got {v!r}"
def test_resolve_auto_uses_globals_for_custom_provider(self):
"""_resolve_auto reads base_url/api_key from globals when main_runtime is None."""
import agent.auxiliary_client as mod
mod.clear_runtime_main()
try:
mod.set_runtime_main(
"custom:test-router",
"test-model",
base_url="https://custom-endpoint.example.com/v1",
api_key="sk-test-123",
)
with patch.object(mod, "resolve_provider_client") as mock_resolve:
mock_resolve.return_value = (MagicMock(), "test-model")
client, resolved = mod._resolve_auto(main_runtime=None)
mock_resolve.assert_called_once()
call_args = mock_resolve.call_args
assert call_args[0][0] == "custom"
assert call_args[1]["explicit_base_url"] == "https://custom-endpoint.example.com/v1"
assert call_args[1]["explicit_api_key"] == "sk-test-123"
finally:
mod.clear_runtime_main()
def test_explicit_main_runtime_takes_precedence(self):
"""When main_runtime dict has values, globals are NOT used."""
import agent.auxiliary_client as mod
mod.clear_runtime_main()
try:
mod.set_runtime_main(
"custom:router-a",
"model-a",
base_url="https://from-global.example.com",
api_key="sk-global",
)
with patch.object(mod, "resolve_provider_client") as mock_resolve:
mock_resolve.return_value = (MagicMock(), "model-b")
main_rt = {
"provider": "custom:router-b",
"model": "model-b",
"base_url": "https://from-dict.example.com",
"api_key": "sk-dict",
}
mod._resolve_auto(main_runtime=main_rt)
call_args = mock_resolve.call_args[1]
assert call_args["explicit_base_url"] == "https://from-dict.example.com"
assert call_args["explicit_api_key"] == "sk-dict"
finally:
mod.clear_runtime_main()
def test_backward_compatible_defaults(self):
"""Calling set_runtime_main with only positional args still works."""
import agent.auxiliary_client as mod
mod.clear_runtime_main()
try:
mod.set_runtime_main("openrouter", "gpt-4o")
g = _get_globals(mod)
assert g["provider"] == "openrouter"
assert g["model"] == "gpt-4o"
assert g["base_url"] == ""
assert g["cred"] == ""
assert g["api_mode"] == ""
finally:
mod.clear_runtime_main()
class TestResolveAutoCustomEndToEnd:
"""End-to-end routing assertions — build a *real* client (no mock on
resolve_provider_client) and verify the auxiliary auto-detect chain lands
on the user's custom endpoint instead of falling through to the aggregator
chain. These guard the actual user-visible symptom in #34777 (aux tasks
silently routed to a fallback provider) rather than just the wiring.
"""
@staticmethod
def _client_base_url(client):
for chain in (("base_url",), ("_client", "base_url")):
obj = client
try:
for attr in chain:
obj = getattr(obj, attr)
return str(obj)
except AttributeError:
continue
return None
def test_config_less_custom_endpoint_routes_via_global(self, tmp_path, monkeypatch):
"""custom:<name> with NO config entry: the live base_url carried by
set_runtime_main() must build a real client at that endpoint — not
fall through to Step 2 (the regression in #34777)."""
import agent.auxiliary_client as mod
# Hermetic: no aggregator creds, no stale OPENAI_BASE_URL.
for var in ("OPENROUTER_API_KEY", "NOUS_API_KEY", "OPENAI_API_KEY",
"OPENAI_BASE_URL"):
monkeypatch.delenv(var, raising=False)
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
(hermes_home / "config.yaml").write_text(
"model:\n"
" default: glm-5.1\n"
" provider: 'custom:ephemeral'\n"
" base_url: ''\n"
)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
mod.clear_runtime_main()
try:
mod.set_runtime_main(
"custom:ephemeral",
"glm-5.1",
base_url="https://ephemeral.live/v1",
api_key="sk-live",
)
client, resolved = mod.resolve_provider_client("auto", None)
assert client is not None, (
"config-less custom endpoint fell through to Step 2 — "
"the #34777 bug is back"
)
assert resolved == "glm-5.1"
base = self._client_base_url(client)
assert base and base.rstrip("/") == "https://ephemeral.live/v1"
finally:
mod.clear_runtime_main()
def test_named_custom_with_config_entry_still_routes(self, tmp_path, monkeypatch):
"""Regression guard: custom:<name> WITH a custom_providers entry must
still resolve to that entry's endpoint. An earlier competing fix
collapsed the provider to bare ``custom`` before resolution, which
broke the named-custom branch and returned None here."""
import agent.auxiliary_client as mod
for var in ("OPENROUTER_API_KEY", "NOUS_API_KEY", "OPENAI_API_KEY",
"OPENAI_BASE_URL"):
monkeypatch.delenv(var, raising=False)
hermes_home = tmp_path / ".hermes"
hermes_home.mkdir()
(hermes_home / "config.yaml").write_text(
"model:\n"
" default: glm-5.1\n"
" provider: 'custom:openclaw'\n"
" base_url: ''\n"
"custom_providers:\n"
" - name: openclaw\n"
" base_url: 'https://withcfg.example/v1'\n"
" model: glm-5.1\n"
" api_key: cfg-key\n"
)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
# No live base_url carried — resolution must come from config alone,
# via the named-custom branch in resolve_provider_client.
mod.clear_runtime_main()
try:
mod.set_runtime_main("custom:openclaw", "glm-5.1")
client, resolved = mod.resolve_provider_client("auto", None)
assert client is not None
base = self._client_base_url(client)
assert base and base.rstrip("/") == "https://withcfg.example/v1"
finally:
mod.clear_runtime_main()

3
tests/agent/test_shell_hooks.py
View file

@ -9,10 +9,7 @@ covered in ``test_shell_hooks_consent.py``.
from __future__ import annotations
import json
import os
import stat
from pathlib import Path
from typing import Any, Dict
import pytest

1
tests/agent/test_shell_hooks_consent.py
View file

@ -7,7 +7,6 @@ hooks_auto_accept: config key).
from __future__ import annotations
import json
from pathlib import Path
from unittest.mock import patch

1
tests/agent/test_skill_bundles.py
View file

@ -2,7 +2,6 @@
import os
from pathlib import Path
from unittest.mock import patch
import pytest

Some files were not shown because too many files have changed in this diff Show more