Merge pull request #55413 from NousResearch/bb/pre-stop-hook
feat(agent): add pre_verify hook and coding guidance config
This commit is contained in:
brooklyn!
GitHub
commit
a81c5922a2
16 changed files with 537 additions and 1 deletions
|
|
@ -353,6 +353,29 @@ def _coding_mode(config: Optional[dict[str, Any]]) -> str:
|
|||
return "auto"
|
||||
|
||||
|
||||
def _coding_instructions(config: Optional[dict[str, Any]]) -> str:
|
||||
"""Standing operator instructions for the coding posture (config).
|
||||
|
||||
``agent.coding_instructions`` — a string or list of strings appended to the
|
||||
coding brief as an extra stable system block, so a user can pin project-wide
|
||||
coding-workflow rules (e.g. "for UI work don't run tsc/lint until I approve;
|
||||
clean the diff before committing") without editing the shipped brief.
|
||||
Cache-safe: resolved once per session into the stable system-prompt tier,
|
||||
like the rest of the posture.
|
||||
"""
|
||||
if config is None:
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
config = load_config()
|
||||
except Exception:
|
||||
config = {}
|
||||
raw = ((config or {}).get("agent", {}) or {}).get("coding_instructions", "")
|
||||
if isinstance(raw, (list, tuple)):
|
||||
return "\n".join(str(item).strip() for item in raw if str(item).strip())
|
||||
return str(raw or "").strip()
|
||||
|
||||
|
||||
def _resolve_cwd(cwd: Optional[str | Path]) -> Path:
|
||||
if cwd:
|
||||
return Path(cwd).expanduser()
|
||||
|
|
@ -459,6 +482,9 @@ class RuntimeMode:
|
|||
# only to steer edit-format guidance toward the model's family — see
|
||||
# ``_edit_format_line``. Fixed for the session, so cache-safe.
|
||||
model: Optional[str] = None
|
||||
# Standing operator instructions (``agent.coding_instructions``), appended
|
||||
# as an extra stable system block. Empty unless the user configures it.
|
||||
instructions: str = ""
|
||||
|
||||
@property
|
||||
def kind(self) -> str:
|
||||
|
|
@ -505,6 +531,10 @@ class RuntimeMode:
|
|||
workspace = build_coding_workspace_block(self.cwd)
|
||||
if workspace:
|
||||
blocks.append(workspace)
|
||||
# Operator instructions ride their own block so the brief (block 0) stays
|
||||
# byte-stable and cache-keyed independently of user config.
|
||||
if self.instructions:
|
||||
blocks.append(f"Operator instructions (from config):\n{self.instructions}")
|
||||
return blocks
|
||||
|
||||
def compact_skill_categories(self) -> frozenset[str]:
|
||||
|
|
@ -557,6 +587,7 @@ def resolve_runtime_mode(
|
|||
cwd=resolved_cwd,
|
||||
config_mode=mode,
|
||||
model=model,
|
||||
instructions=_coding_instructions(config),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4810,6 +4810,55 @@ def run_conversation(
|
|||
agent._verification_stop_nudges)
|
||||
continue
|
||||
|
||||
# User verification-loop gate: when the agent edited code this
|
||||
# turn, let a registered `pre_verify` hook (plugin/shell) keep it
|
||||
# going one more turn. The shipped guidance is folded into the
|
||||
# evidence-based verify-on-stop nudge above, so this path has no
|
||||
# default continuation cost.
|
||||
_verify_nudge2 = None
|
||||
_edited = sorted(getattr(agent, "_turn_file_mutation_paths", set()) or [])
|
||||
_attempt = getattr(agent, "_pre_verify_nudges", 0)
|
||||
try:
|
||||
from agent.verify_hooks import max_verify_nudges
|
||||
from hermes_cli.plugins import get_pre_verify_continue_message, has_hook
|
||||
|
||||
if _edited and has_hook("pre_verify") and _attempt < max_verify_nudges():
|
||||
# Posture is fixed for the session — resolve once + cache.
|
||||
coding = getattr(agent, "_resolved_is_coding", None)
|
||||
if coding is None:
|
||||
from agent.coding_context import is_coding_context
|
||||
coding = bool(is_coding_context(platform=getattr(agent, "platform", "") or ""))
|
||||
agent._resolved_is_coding = coding
|
||||
_verify_nudge2 = get_pre_verify_continue_message(
|
||||
session_id=getattr(agent, "session_id", None) or "",
|
||||
platform=getattr(agent, "platform", "") or "",
|
||||
model=getattr(agent, "model", "") or "",
|
||||
coding=coding,
|
||||
attempt=_attempt,
|
||||
final_response=final_response,
|
||||
changed_paths=_edited,
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("pre_verify hook check failed", exc_info=True)
|
||||
_verify_nudge2 = None
|
||||
|
||||
if _verify_nudge2:
|
||||
agent._pre_verify_nudges = _attempt + 1
|
||||
final_msg["finish_reason"] = "verify_hook_continue"
|
||||
# Same alternation contract as verify-on-stop: keep the
|
||||
# attempted answer in history, follow it with a synthetic
|
||||
# user nudge, and don't surface the premature answer.
|
||||
messages.append(final_msg)
|
||||
messages.append({
|
||||
"role": "user",
|
||||
"content": _verify_nudge2,
|
||||
"_pre_verify_synthetic": True,
|
||||
})
|
||||
agent._session_messages = messages
|
||||
logger.debug("pre_verify nudge issued (attempt %d)",
|
||||
agent._pre_verify_nudges)
|
||||
continue
|
||||
|
||||
messages.append(final_msg)
|
||||
|
||||
_turn_exit_reason = f"text_response(finish_reason={finish_reason})"
|
||||
|
|
|
|||
|
|
@ -588,6 +588,17 @@ def _parse_response(event: str, stdout: str) -> Optional[Dict[str, Any]]:
|
|||
return {"action": "block", "message": _block_message(data.get("reason"), data.get("message"))}
|
||||
return None
|
||||
|
||||
if event == "pre_verify":
|
||||
# "continue" (Hermes) / "block" (Claude-Code Stop: block the stop) both
|
||||
# mean keep going; the message/reason is the follow-up for the model. A
|
||||
# continue with no message is a no-op — let the turn finish.
|
||||
action = str(data.get("action") or data.get("decision") or "").strip().lower()
|
||||
if action in {"continue", "block"}:
|
||||
message = data.get("message") or data.get("reason")
|
||||
if isinstance(message, str) and message.strip():
|
||||
return {"action": "continue", "message": message.strip()}
|
||||
return None
|
||||
|
||||
context = data.get("context")
|
||||
if isinstance(context, str) and context.strip():
|
||||
return {"context": context}
|
||||
|
|
|
|||
|
|
@ -443,6 +443,7 @@ def build_turn_context(
|
|||
agent._turn_failed_file_mutations = {}
|
||||
agent._turn_file_mutation_paths = set()
|
||||
agent._verification_stop_nudges = 0
|
||||
agent._pre_verify_nudges = 0
|
||||
|
||||
# Record the execution thread so interrupt()/clear_interrupt() can scope
|
||||
# the tool-level interrupt signal to THIS agent's thread only.
|
||||
|
|
|
|||
|
|
@ -273,6 +273,15 @@ def build_verify_on_stop_nudge(
|
|||
if state == "passed":
|
||||
return None
|
||||
|
||||
# Optional shipped coding guidance, only paid when this evidence gate fires.
|
||||
try:
|
||||
from agent.verify_hooks import coding_verify_guidance
|
||||
|
||||
guidance = coding_verify_guidance()
|
||||
except Exception:
|
||||
guidance = None
|
||||
addendum = f"\n\n{guidance}" if guidance else ""
|
||||
|
||||
if verify_commands:
|
||||
command_instruction = (
|
||||
"Run the relevant verification command now ("
|
||||
|
|
@ -297,7 +306,8 @@ def build_verify_on_stop_nudge(
|
|||
f"Verification status: {_status_detail(status)}\n\n"
|
||||
f"Changed paths:\n{_format_changed_paths(paths)}\n\n"
|
||||
f"{command_instruction} If verification is not possible, explain the "
|
||||
"concrete blocker instead of claiming the work is fully verified.]"
|
||||
"concrete blocker instead of claiming the work is fully verified."
|
||||
f"{addendum}]"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
69
agent/verify_hooks.py
Normal file
69
agent/verify_hooks.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
"""Verification-loop helpers for the ``pre_verify`` round-end gate.
|
||||
|
||||
When the agent has edited code and is about to verify/finish, the loop fires the
|
||||
``pre_verify`` hook (user directives resolved by
|
||||
:func:`hermes_cli.plugins.get_pre_verify_continue_message`). A directive keeps
|
||||
the agent going one more turn — run a check, defer it, tidy the diff — instead of
|
||||
stopping immediately.
|
||||
|
||||
The shipped coding guidance lives on the evidence-based verification-stop nudge
|
||||
(``agent/verification_stop.py``), not as a second default stop gate. That keeps
|
||||
the default token cost tied to the existing "missing verification evidence"
|
||||
decision while preserving ``pre_verify`` for user/plugin policy.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from utils import is_truthy_value
|
||||
|
||||
DEFAULT_MAX_VERIFY_NUDGES = 3
|
||||
|
||||
# Shipped guidance appended to the verification-stop nudge when code lacks fresh
|
||||
# verification evidence. Wording mirrors the user-facing "clean your work"
|
||||
# workflow, but does not create its own extra model turn.
|
||||
CODING_VERIFY_GUIDANCE = (
|
||||
"[Coding] Before you run tests/linters or call this done: if this is "
|
||||
"creative UI/visual work, hold off on tests and linters until the user says "
|
||||
"they like the result or you're about to commit. And before every commit, "
|
||||
"clean your work: keep it KISS/DRY, match the surrounding code style, and be "
|
||||
"elitist, shorthand, clever, concise, efficient, and elegant."
|
||||
)
|
||||
|
||||
|
||||
def max_verify_nudges(config: Optional[dict[str, Any]] = None) -> int:
|
||||
"""Bound on consecutive ``pre_verify`` continue directives per turn (>= 0)."""
|
||||
agent_cfg = _agent_cfg(config)
|
||||
raw = agent_cfg.get("max_verify_nudges")
|
||||
try:
|
||||
return max(0, int(raw))
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_MAX_VERIFY_NUDGES
|
||||
|
||||
|
||||
def coding_verify_guidance(config: Optional[dict[str, Any]] = None) -> Optional[str]:
|
||||
"""Return the optional guidance appended to verification-stop nudges."""
|
||||
if not is_truthy_value(_agent_cfg(config).get("verify_guidance", True), default=True):
|
||||
return None
|
||||
return CODING_VERIFY_GUIDANCE
|
||||
|
||||
|
||||
def _agent_cfg(config: Optional[dict[str, Any]]) -> dict[str, Any]:
|
||||
if config is None:
|
||||
try:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
config = load_config()
|
||||
except Exception:
|
||||
config = {}
|
||||
agent_cfg = (config or {}).get("agent") if isinstance(config, dict) else None
|
||||
return agent_cfg if isinstance(agent_cfg, dict) else {}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CODING_VERIFY_GUIDANCE",
|
||||
"DEFAULT_MAX_VERIFY_NUDGES",
|
||||
"coding_verify_guidance",
|
||||
"max_verify_nudges",
|
||||
]
|
||||
|
|
@ -646,6 +646,25 @@ agent:
|
|||
# force it on or off; the HERMES_VERIFY_ON_STOP env var (1/0) takes precedence.
|
||||
# verify_on_stop: auto
|
||||
|
||||
# Standing operator instructions for the coding posture (when Hermes is in a
|
||||
# code workspace). Appended to the coding brief as an extra system block, so
|
||||
# you can pin project-wide workflow rules without editing the shipped brief.
|
||||
# Accepts a string or a list of strings. Takes effect next session.
|
||||
# coding_instructions:
|
||||
# - "For UI work, don't run tsc/lint until I approve the look."
|
||||
# - "Clean the diff before you commit and push."
|
||||
|
||||
# When verify-on-stop finds edited code without fresh verification evidence,
|
||||
# append guidance for creative UI work (avoid broad tsc/lint/test before visual
|
||||
# approval) and clean-diff expectations. Set false to keep that nudge terse.
|
||||
# verify_guidance: true
|
||||
|
||||
# A `pre_verify` hook (plugin or shell, see Event Hooks docs) can keep the
|
||||
# agent going one more turn to verify/clean before finishing. This caps how
|
||||
# many times one turn may be nudged to continue, so a hook can't trap the loop.
|
||||
# Default 3.
|
||||
# max_verify_nudges: 3
|
||||
|
||||
# Enable verbose logging
|
||||
verbose: false
|
||||
|
||||
|
|
|
|||
|
|
@ -999,6 +999,21 @@ DEFAULT_CONFIG = {
|
|||
# "on" — force the prompt posture everywhere.
|
||||
# "off" — disable entirely.
|
||||
"coding_context": "auto",
|
||||
# Standing operator instructions for the coding posture. A string (or
|
||||
# list of strings) appended to the coding brief as an extra stable
|
||||
# system block — pin project-wide workflow rules here instead of editing
|
||||
# the shipped brief, e.g. "For UI work, don't run tsc/lint until I
|
||||
# approve. Clean the diff before you commit and push." Cache-safe:
|
||||
# takes effect next session. Empty by default.
|
||||
"coding_instructions": "",
|
||||
# When verify-on-stop finds edited code without fresh verification
|
||||
# evidence, append guidance for creative UI work (avoid broad
|
||||
# tsc/lint/test before visual approval) and clean-diff expectations.
|
||||
# Set false to keep the evidence nudge terse.
|
||||
"verify_guidance": True,
|
||||
# Upper bound on consecutive `pre_verify` "continue" nudges in a single
|
||||
# turn, so a user/plugin hook can never trap the loop.
|
||||
"max_verify_nudges": 3,
|
||||
# Verification closure: after the agent edits files in a code workspace,
|
||||
# do not accept a final answer until fresh verification evidence exists
|
||||
# or the agent explains why it cannot run checks. The loop is bounded
|
||||
|
|
|
|||
|
|
@ -139,6 +139,15 @@ _DEFAULT_PAYLOADS = {
|
|||
"model": "gpt-4",
|
||||
"platform": "cli",
|
||||
},
|
||||
"pre_verify": {
|
||||
"session_id": "test-session",
|
||||
"platform": "cli",
|
||||
"model": "gpt-4",
|
||||
"coding": True,
|
||||
"attempt": 0,
|
||||
"final_response": "All done — the change is applied.",
|
||||
"changed_paths": ["src/app.tsx"],
|
||||
},
|
||||
"on_session_start": {"session_id": "test-session"},
|
||||
"on_session_end": {"session_id": "test-session"},
|
||||
"on_session_finalize": {"session_id": "test-session"},
|
||||
|
|
|
|||
|
|
@ -136,6 +136,17 @@ VALID_HOOKS: Set[str] = {
|
|||
"transform_llm_output",
|
||||
"pre_llm_call",
|
||||
"post_llm_call",
|
||||
# Verification-loop gate. Fired once per turn when the agent has edited code
|
||||
# and is about to verify/finish (after the verify-on-stop guard). A callback
|
||||
# may keep the agent going — run a check, defer it, tidy the diff — instead
|
||||
# of stopping by returning:
|
||||
# {"action": "continue", "message": "<follow-up instruction>"}
|
||||
# The Claude-Code Stop shape {"decision": "block", "reason": "..."} (block
|
||||
# the stop == keep going) is accepted too. Anything else lets the turn
|
||||
# finish. Hermes' shipped guidance lives in the evidence-based
|
||||
# verification-stop nudge; this hook is for user/plugin policy and is
|
||||
# bounded by agent.max_verify_nudges.
|
||||
"pre_verify",
|
||||
"pre_api_request",
|
||||
"post_api_request",
|
||||
"api_request_error",
|
||||
|
|
@ -2029,6 +2040,57 @@ def get_pre_tool_call_block_message(
|
|||
return None
|
||||
|
||||
|
||||
def get_pre_verify_continue_message(
|
||||
*,
|
||||
session_id: str = "",
|
||||
platform: str = "",
|
||||
model: str = "",
|
||||
coding: bool = False,
|
||||
attempt: int = 0,
|
||||
final_response: str = "",
|
||||
changed_paths: Optional[List[str]] = None,
|
||||
) -> Optional[str]:
|
||||
"""Check user ``pre_verify`` hooks for a directive to keep the agent going.
|
||||
|
||||
Fired once per turn when the agent edited code and is about to verify/finish.
|
||||
A hook keeps the turn going (run a check, defer it, tidy the diff) by
|
||||
returning::
|
||||
|
||||
{"action": "continue", "message": "<follow-up for the model>"}
|
||||
|
||||
The Claude-Code Stop shape ``{"decision": "block", "reason": "..."}`` (block
|
||||
the stop == keep going) is accepted too. The first directive carrying a
|
||||
non-empty message wins; any other return lets the turn finish. Mirrors
|
||||
:func:`get_pre_tool_call_block_message` — the call site stays a one-liner.
|
||||
|
||||
``coding`` / ``attempt`` let a hook scope itself (``if not coding`` …) and
|
||||
self-throttle (``if attempt`` …), the same way a ``pre_tool_call`` hook
|
||||
scopes on ``tool_name``.
|
||||
"""
|
||||
hook_results = invoke_hook(
|
||||
"pre_verify",
|
||||
session_id=session_id,
|
||||
platform=platform,
|
||||
model=model,
|
||||
coding=coding,
|
||||
attempt=attempt,
|
||||
final_response=final_response,
|
||||
changed_paths=list(changed_paths or []),
|
||||
)
|
||||
|
||||
for result in hook_results:
|
||||
if not isinstance(result, dict):
|
||||
continue
|
||||
action = str(result.get("action") or result.get("decision") or "").strip().lower()
|
||||
if action not in ("continue", "block"):
|
||||
continue
|
||||
message = result.get("message") or result.get("reason")
|
||||
if isinstance(message, str) and message.strip():
|
||||
return message.strip()
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_plugins_discovered(force: bool = False) -> PluginManager:
|
||||
"""Return the global manager after ensuring plugin discovery has run.
|
||||
|
||||
|
|
|
|||
|
|
@ -353,6 +353,39 @@ class TestRuntimeMode:
|
|||
assert any("coding agent" in b for b in blocks)
|
||||
assert any("Workspace" in b for b in blocks)
|
||||
|
||||
def test_coding_instructions_append_their_own_block(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
cfg = {
|
||||
"agent": {
|
||||
"coding_context": "on",
|
||||
"coding_instructions": "Clean the diff before commit.",
|
||||
}
|
||||
}
|
||||
mode = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config=cfg)
|
||||
blocks = mode.system_blocks()
|
||||
# The brief stays block 0 (byte-stable, cache-keyed independently); the
|
||||
# operator instructions ride a separate trailing block.
|
||||
assert blocks[0] == cc.CODING_AGENT_GUIDANCE
|
||||
assert any("Clean the diff before commit." in b for b in blocks[1:])
|
||||
|
||||
def test_coding_instructions_accept_a_list(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
cfg = {
|
||||
"agent": {
|
||||
"coding_context": "on",
|
||||
"coding_instructions": ["No tsc/lint on UI.", "Clean the diff."],
|
||||
}
|
||||
}
|
||||
mode = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config=cfg)
|
||||
instr_block = mode.system_blocks()[-1]
|
||||
assert "No tsc/lint on UI." in instr_block
|
||||
assert "Clean the diff." in instr_block
|
||||
|
||||
def test_no_instructions_block_when_unset(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
mode = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={"agent": {"coding_context": "on"}})
|
||||
assert not any("Operator instructions" in b for b in mode.system_blocks())
|
||||
|
||||
def test_toolset_selection_gated_on_focus(self, tmp_path):
|
||||
_git_init(tmp_path)
|
||||
focus = cc.resolve_runtime_mode(platform="cli", cwd=tmp_path, config={"agent": {"coding_context": "focus"}})
|
||||
|
|
|
|||
|
|
@ -97,6 +97,24 @@ class TestParseResponse:
|
|||
)
|
||||
assert r is None
|
||||
|
||||
def test_pre_verify_continue_canonical(self):
|
||||
r = shell_hooks._parse_response(
|
||||
"pre_verify", '{"action": "continue", "message": "run checks"}',
|
||||
)
|
||||
assert r == {"action": "continue", "message": "run checks"}
|
||||
|
||||
def test_pre_verify_block_is_continue_claude_style(self):
|
||||
# Claude-Code Stop hooks: block the stop == keep going; reason → message.
|
||||
r = shell_hooks._parse_response(
|
||||
"pre_verify", '{"decision": "block", "reason": "run the formatter"}',
|
||||
)
|
||||
assert r == {"action": "continue", "message": "run the formatter"}
|
||||
|
||||
def test_pre_verify_without_message_is_noop(self):
|
||||
# A continue with nothing to tell the model lets the turn finish.
|
||||
assert shell_hooks._parse_response("pre_verify", '{"action": "continue"}') is None
|
||||
assert shell_hooks._parse_response("pre_verify", '{"decision": "allow"}') is None
|
||||
|
||||
def test_block_action_without_message_uses_default(self):
|
||||
"""Block is honored even when message/reason is absent."""
|
||||
r = shell_hooks._parse_response("pre_tool_call", '{"action": "block"}')
|
||||
|
|
|
|||
|
|
@ -215,6 +215,7 @@ def test_nudge_after_unverified_edit_with_known_command(tmp_path, monkeypatch):
|
|||
assert "fresh passing verification evidence" in nudge
|
||||
assert "`pnpm run test`" in nudge
|
||||
assert changed in nudge
|
||||
assert "creative UI/visual work" in nudge
|
||||
|
||||
|
||||
def test_nudge_includes_failed_output_summary(tmp_path, monkeypatch):
|
||||
|
|
@ -249,6 +250,23 @@ def test_no_suite_nudge_requests_temp_script(tmp_path, monkeypatch):
|
|||
assert tempfile.gettempdir() in nudge
|
||||
assert "ad-hoc verification" in nudge
|
||||
assert "suite green" in nudge
|
||||
assert "creative UI/visual work" in nudge
|
||||
|
||||
|
||||
def test_verify_guidance_can_be_disabled(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
_node_project(tmp_path)
|
||||
changed = str(tmp_path / "src" / "app.ts")
|
||||
|
||||
from agent import verify_hooks
|
||||
|
||||
monkeypatch.setattr(verify_hooks, "coding_verify_guidance", lambda: None)
|
||||
|
||||
nudge = build_verify_on_stop_nudge(session_id="s1", changed_paths=[changed])
|
||||
|
||||
assert nudge is not None
|
||||
assert "fresh passing verification evidence" in nudge
|
||||
assert "creative UI/visual work" not in nudge
|
||||
|
||||
|
||||
def test_ad_hoc_pass_satisfies_no_suite_stop_loop(tmp_path, monkeypatch):
|
||||
|
|
|
|||
53
tests/agent/test_verify_hooks.py
Normal file
53
tests/agent/test_verify_hooks.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""Unit tests for the verification-loop policy (agent/verify_hooks.py).
|
||||
|
||||
The `pre_verify` user-hook aggregation lives in `hermes_cli.plugins`
|
||||
(`get_pre_verify_continue_message`) and is tested in
|
||||
`tests/hermes_cli/test_plugins.py`, alongside `get_pre_tool_call_block_message`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from agent import verify_hooks
|
||||
|
||||
|
||||
class TestMaxVerifyNudges:
|
||||
def test_default_when_unset(self):
|
||||
assert (
|
||||
verify_hooks.max_verify_nudges({})
|
||||
== verify_hooks.DEFAULT_MAX_VERIFY_NUDGES
|
||||
)
|
||||
assert (
|
||||
verify_hooks.max_verify_nudges({"agent": {}})
|
||||
== verify_hooks.DEFAULT_MAX_VERIFY_NUDGES
|
||||
)
|
||||
|
||||
def test_reads_and_coerces(self):
|
||||
assert verify_hooks.max_verify_nudges({"agent": {"max_verify_nudges": 5}}) == 5
|
||||
assert verify_hooks.max_verify_nudges({"agent": {"max_verify_nudges": "2"}}) == 2
|
||||
assert verify_hooks.max_verify_nudges({"agent": {"max_verify_nudges": -1}}) == 0
|
||||
|
||||
def test_bad_value_falls_back(self):
|
||||
assert (
|
||||
verify_hooks.max_verify_nudges({"agent": {"max_verify_nudges": "x"}})
|
||||
== verify_hooks.DEFAULT_MAX_VERIFY_NUDGES
|
||||
)
|
||||
|
||||
|
||||
class TestCodingVerifyGuidance:
|
||||
def test_enabled_by_default(self):
|
||||
assert (
|
||||
verify_hooks.coding_verify_guidance({})
|
||||
== verify_hooks.CODING_VERIFY_GUIDANCE
|
||||
)
|
||||
assert (
|
||||
verify_hooks.coding_verify_guidance({"agent": {}})
|
||||
== verify_hooks.CODING_VERIFY_GUIDANCE
|
||||
)
|
||||
|
||||
def test_reads_truthy_config(self):
|
||||
cfg = {"agent": {"verify_guidance": "yes"}}
|
||||
assert verify_hooks.coding_verify_guidance(cfg) == verify_hooks.CODING_VERIFY_GUIDANCE
|
||||
|
||||
def test_opt_out_via_config(self):
|
||||
off = {"agent": {"verify_guidance": False}}
|
||||
assert verify_hooks.coding_verify_guidance(off) is None
|
||||
|
|
@ -18,6 +18,7 @@ from hermes_cli.plugins import (
|
|||
get_plugin_command_handler,
|
||||
get_plugin_commands,
|
||||
get_pre_tool_call_block_message,
|
||||
get_pre_verify_continue_message,
|
||||
has_middleware,
|
||||
resolve_plugin_command_result,
|
||||
)
|
||||
|
|
@ -858,6 +859,73 @@ class TestPreToolCallBlocking:
|
|||
assert get_pre_tool_call_block_message("terminal", {}) == "first blocker"
|
||||
|
||||
|
||||
class TestGetPreVerifyContinueMessage:
|
||||
"""`pre_verify` directive aggregation — mirrors the pre_tool_call block path."""
|
||||
|
||||
def test_continue_canonical(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [{"action": "continue", "message": "run checks"}],
|
||||
)
|
||||
assert get_pre_verify_continue_message(session_id="s") == "run checks"
|
||||
|
||||
def test_claude_block_means_continue(self, monkeypatch):
|
||||
# Claude-Code Stop: "block" the stop == keep going; reason → message.
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [{"decision": "block", "reason": "run the formatter"}],
|
||||
)
|
||||
assert get_pre_verify_continue_message() == "run the formatter"
|
||||
|
||||
def test_first_actionable_directive_wins(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [
|
||||
"noise", # not a dict
|
||||
{"action": "continue"}, # no message → skipped
|
||||
{"action": "continue", "message": "second"},
|
||||
{"action": "continue", "message": "third"},
|
||||
],
|
||||
)
|
||||
assert get_pre_verify_continue_message() == "second"
|
||||
|
||||
def test_message_is_trimmed(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [{"action": "continue", "message": " tidy up "}],
|
||||
)
|
||||
assert get_pre_verify_continue_message() == "tidy up"
|
||||
|
||||
def test_invalid_returns_ignored(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.plugins.invoke_hook",
|
||||
lambda hook_name, **kwargs: [
|
||||
{"action": "allow"}, # wrong action
|
||||
{"context": "noise"}, # not a directive
|
||||
{"action": "continue", "message": " "}, # blank message
|
||||
{"action": "continue", "message": 42}, # message not str
|
||||
],
|
||||
)
|
||||
assert get_pre_verify_continue_message() is None
|
||||
|
||||
def test_none_when_no_hooks(self, monkeypatch):
|
||||
monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda hook_name, **kwargs: [])
|
||||
assert get_pre_verify_continue_message() is None
|
||||
|
||||
def test_forwards_scope_signals_to_hooks(self, monkeypatch):
|
||||
seen = {}
|
||||
|
||||
def capture(hook_name, **kwargs):
|
||||
seen.update(kwargs)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr("hermes_cli.plugins.invoke_hook", capture)
|
||||
get_pre_verify_continue_message(coding=True, attempt=2, changed_paths=["a.py"])
|
||||
assert seen["coding"] is True
|
||||
assert seen["attempt"] == 2
|
||||
assert seen["changed_paths"] == ["a.py"]
|
||||
|
||||
|
||||
class TestThreadToolWhitelist:
|
||||
"""Tests for the thread-local tool whitelist used by background review forks."""
|
||||
|
||||
|
|
|
|||
|
|
@ -382,6 +382,7 @@ def register(ctx):
|
|||
| [`post_tool_call`](#post_tool_call) | After any tool returns | ignored |
|
||||
| [`pre_llm_call`](#pre_llm_call) | Once per turn, before the tool-calling loop | `{"context": str}` to prepend context to the user message |
|
||||
| [`post_llm_call`](#post_llm_call) | Once per turn, after the tool-calling loop | ignored |
|
||||
| [`pre_verify`](#pre_verify) | Once per turn when the agent edited code, before it verifies/finishes | `{"action": "continue", "message": str}` to keep going |
|
||||
| [`on_session_start`](#on_session_start) | New session created (first turn only) | ignored |
|
||||
| [`on_session_end`](#on_session_end) | Session ends | ignored |
|
||||
| [`on_session_finalize`](#on_session_finalize) | CLI/gateway tears down an active session (flush, save, stats) | ignored |
|
||||
|
|
@ -652,6 +653,71 @@ def register(ctx):
|
|||
|
||||
---
|
||||
|
||||
### `pre_verify`
|
||||
|
||||
Fires **once per turn when the agent edited code**, just before it finishes (after the built-in verify-on-stop guard). This is a user/plugin policy gate: a callback can keep the agent going — run a check, defer it, tidy the diff — instead of letting it stop.
|
||||
|
||||
Hermes' shipped verification guidance is not a default `pre_verify` hook. It is appended to the evidence-based verify-on-stop nudge when edited code lacks fresh verification evidence, so it does not create a second default continuation path. Set `agent.verify_guidance: false` to keep that built-in evidence nudge terse.
|
||||
|
||||
**Callback signature:**
|
||||
|
||||
```python
|
||||
def my_callback(session_id: str, platform: str, model: str, coding: bool,
|
||||
attempt: int, final_response: str, changed_paths: list, **kwargs):
|
||||
```
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `session_id` | `str` | Unique identifier for the current session |
|
||||
| `platform` | `str` | Where the session is running (`"cli"`, `"telegram"`, …) |
|
||||
| `model` | `str` | The model identifier |
|
||||
| `coding` | `bool` | Whether the turn is in the coding posture (in a code workspace) — scope your hook on this |
|
||||
| `attempt` | `int` | How many times this turn has already been nudged (0 on the first) — self-throttle on this |
|
||||
| `final_response` | `str` | The answer the agent is about to deliver |
|
||||
| `changed_paths` | `list` | Files the agent edited this turn (sorted, always non-empty here) |
|
||||
|
||||
Scope a hook to the coding context by checking `coding` and make it one-shot with `attempt` (shell hooks read both from `.extra`), the same way a `pre_tool_call` hook scopes on `tool_name` — so you can register several `pre_verify` hooks, each firing only where it should.
|
||||
|
||||
**Fires:** In `agent/conversation_loop.py`, at the point the agent would accept a final answer, immediately after the verify-on-stop check — but only when the agent edited code this turn and at least one `pre_verify` hook is registered.
|
||||
|
||||
**Return value — keep the agent going:**
|
||||
|
||||
```python
|
||||
return {"action": "continue", "message": "Run the formatter on your changes, then finish."}
|
||||
```
|
||||
|
||||
The `message` is appended as a synthetic user turn and the loop runs again. The Claude-Code Stop shape (`{"decision": "block", "reason": "..."}`, where blocking the stop means *keep going*) is accepted too. A directive with no message — or any other return — lets the turn finish.
|
||||
|
||||
**Bounded:** consecutive continue directives in one turn are capped by `agent.max_verify_nudges` (default 3), so a hook that always says continue can never trap the loop. The attempted answer is kept in history but not surfaced to the user while the agent is being nudged.
|
||||
|
||||
**Make it idempotent:** the hook re-fires after each nudge, so gate on `attempt` (`if attempt: return None`) — otherwise it just nudges until the bound is hit.
|
||||
|
||||
**Use cases:** defer tests/lints during creative iteration, require green checks for certain paths, block "done" until a changelog entry exists, run a project-specific verification checklist.
|
||||
|
||||
**Example — defer checks on creative UI work, scoped + one-shot:**
|
||||
|
||||
```python
|
||||
UI = (".tsx", ".jsx", ".css", ".scss")
|
||||
|
||||
def defer_ui_checks(coding, attempt, changed_paths, **kwargs):
|
||||
if attempt or not coding:
|
||||
return None # one-shot, coding only
|
||||
if not all(p.endswith(UI) for p in changed_paths):
|
||||
return None # only pure-UI edits
|
||||
return {
|
||||
"action": "continue",
|
||||
"message": "This is UI work — don't run tests/lints yet; ask the user to "
|
||||
"eyeball it first, and clean the diff before any commit.",
|
||||
}
|
||||
|
||||
def register(ctx):
|
||||
ctx.register_hook("pre_verify", defer_ui_checks)
|
||||
```
|
||||
|
||||
For standing guidance that should shape the built-in missing-evidence nudge, use `agent.verify_guidance`. For broader coding posture rules that don't need to *gate* verification, prefer `agent.coding_instructions` in `config.yaml` — it rides the coding brief and costs no extra turn.
|
||||
|
||||
---
|
||||
|
||||
### `on_session_start`
|
||||
|
||||
Fires **once** when a brand-new session is created. Does **not** fire on session continuation (when the user sends a second message in an existing session).
|
||||
|
|
@ -1284,6 +1350,10 @@ Each time the event fires, Hermes spawns a subprocess for every matching hook (m
|
|||
// Inject context for pre_llm_call:
|
||||
{"context": "Today is Friday, 2026-04-17"}
|
||||
|
||||
// Keep the agent going at the verify gate (pre_verify); both shapes accepted:
|
||||
{"action": "continue", "message": "Run the formatter, then finish."}
|
||||
{"decision": "block", "reason": "Run the formatter, then finish."}
|
||||
|
||||
// Silent no-op — any empty / non-matching output is fine:
|
||||
```
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue