hermes-agent/tests/agent/test_moa_trace_streamed_capture.py

"""Tests for MoA trace aggregator-output capture across streaming modes.

The MoA full-turn trace (opt-in ``moa.save_traces``) must record the
aggregator's acting output whether the aggregator ran non-streaming (inline
capture at call time) or streaming (captured after the fact from the caller's
resolved assistant text). Before the streamed-capture fix, a streamed
aggregator left ``output: null`` in the trace and only pointed at state.db,
so an offline audit of a benchmark run (which drives the streaming display
path via ``hermes chat --query``) couldn't see what the aggregator actually
produced without joining to the session DB by hand.

These exercise the real ``consume_and_save_trace`` → ``save_moa_turn`` path
with real file I/O against a temp HERMES_HOME — no mocks on the write path.
"""

from __future__ import annotations

import json

import pytest

from agent.moa_loop import MoAChatCompletions


def _enable_traces(tmp_path, monkeypatch):
    """Point HERMES_HOME at a temp dir and turn moa.save_traces on."""
    hermes_home = tmp_path / ".hermes"
    hermes_home.mkdir()
    monkeypatch.setenv("HERMES_HOME", str(hermes_home))

    # save_moa_turn reads config via hermes_cli.config.load_config; stub it to
    # return traces-on so the test doesn't depend on a real config file.
    import agent.moa_trace as moa_trace

    monkeypatch.setattr(
        moa_trace,
        "load_config",
        lambda: {"moa": {"save_traces": True}},
        raising=False,
    )
    # load_config is imported lazily inside _traces_enabled_and_dir; patch the
    # source module attribute it imports from as well.
    import hermes_cli.config as cfg

    monkeypatch.setattr(
        cfg, "load_config", lambda: {"moa": {"save_traces": True}}, raising=False
    )
    return hermes_home / "moa-traces"


def _make_completions_with_pending(streamed: bool, inline_output):
    """Build a MoAChatCompletions with a pending trace mimicking one turn."""
    mc = MoAChatCompletions.__new__(MoAChatCompletions)
    mc._pending_trace = {
        "preset": "closed",
        "reference_outputs": [],  # references not under test here
        "aggregator_label": "openrouter:anthropic/claude-opus-4.8",
        "aggregator_slot": {
            "model": "anthropic/claude-opus-4.8",
            "provider": "openrouter",
        },
        "aggregator_temperature": 0.4,
        "aggregator_input_messages": [
            {"role": "system", "content": "sys"},
            {"role": "user", "content": "do the thing"},
        ],
        "aggregator_output": inline_output,
        "aggregator_streamed": streamed,
    }
    return mc


def _read_single_trace(trace_dir, session_id):
    path = trace_dir / f"{session_id}.jsonl"
    assert path.exists(), f"trace file not written: {path}"
    lines = path.read_text().strip().split("\n")
    assert len(lines) == 1
    return json.loads(lines[0])


def test_streamed_aggregator_output_captured_from_fallback(tmp_path, monkeypatch):
    """Streaming turn: inline output is None, fallback text is embedded."""
    trace_dir = _enable_traces(tmp_path, monkeypatch)
    mc = _make_completions_with_pending(streamed=True, inline_output=None)

    mc.consume_and_save_trace(
        "sess_streamed",
        aggregator_output_fallback="the acting aggregator answer",
    )

    rec = _read_single_trace(trace_dir, "sess_streamed")
    agg = rec["aggregator"]
    assert agg["streamed"] is True
    assert agg["output"] == "the acting aggregator answer"
    assert agg["output_location"] == "inline_from_stream"


def test_non_streaming_prefers_inline_over_fallback(tmp_path, monkeypatch):
    """Non-streaming turn keeps its inline capture even if a fallback is passed."""
    trace_dir = _enable_traces(tmp_path, monkeypatch)
    mc = _make_completions_with_pending(
        streamed=False, inline_output="inline captured text"
    )

    mc.consume_and_save_trace(
        "sess_inline",
        aggregator_output_fallback="SHOULD NOT BE USED",
    )

    rec = _read_single_trace(trace_dir, "sess_inline")
    agg = rec["aggregator"]
    assert agg["streamed"] is False
    assert agg["output"] == "inline captured text"
    assert agg["output_location"] == "inline"


def test_streamed_without_fallback_points_to_session_db(tmp_path, monkeypatch):
    """Streaming turn with no resolvable text falls back to the state.db pointer."""
    trace_dir = _enable_traces(tmp_path, monkeypatch)
    mc = _make_completions_with_pending(streamed=True, inline_output=None)

    mc.consume_and_save_trace("sess_nofb", aggregator_output_fallback=None)

    rec = _read_single_trace(trace_dir, "sess_nofb")
    agg = rec["aggregator"]
    assert agg["streamed"] is True
    assert agg["output"] is None
    assert agg["output_location"] == "assistant_message_in_session_db"


def test_pending_trace_cleared_after_flush(tmp_path, monkeypatch):
    """A second flush is a no-op (pending cleared) — never double-writes."""
    trace_dir = _enable_traces(tmp_path, monkeypatch)
    mc = _make_completions_with_pending(streamed=True, inline_output=None)

    mc.consume_and_save_trace("sess_once", aggregator_output_fallback="x")
    # Second call: pending is None now, must not append a second line.
    mc.consume_and_save_trace("sess_once", aggregator_output_fallback="y")

    path = trace_dir / "sess_once.jsonl"
    lines = path.read_text().strip().split("\n")
    assert len(lines) == 1


def test_empty_fallback_string_treated_as_missing(tmp_path, monkeypatch):
    """An empty-string fallback must not override to '' — treated as absent."""
    trace_dir = _enable_traces(tmp_path, monkeypatch)
    mc = _make_completions_with_pending(streamed=True, inline_output=None)

    mc.consume_and_save_trace("sess_empty", aggregator_output_fallback="")

    rec = _read_single_trace(trace_dir, "sess_empty")
    agg = rec["aggregator"]
    assert agg["output"] is None
    assert agg["output_location"] == "assistant_message_in_session_db"