fix(state): widen surrogate scrub to remaining raw-str bind sites

Follow-up to the cherry-picked fix: the same UnicodeEncodeError bind
failure was live at sibling sites the PR didn't cover —

- api_content sidecar (append_message, _insert_message_rows,
  set_latest_user_api_content): bound raw; a surrogate in the composed
  api_content aborted the whole row/UPDATE. Scrubbing is wire-accurate:
  the conversation loop already scrubs every outgoing payload, so the
  scrubbed form IS what was sent.
- tool_name (both INSERT sites): raw bind.
- sanitize_title: session titles from LLM title generation or /title
  could carry surrogates; scrub before validation.

E2E-verified each site raised UnicodeEncodeError on main and persists
after this commit. Tests added for all five paths.
This commit is contained in:
Kshitij Kapoor 2026-07-19 13:28:28 +05:30 committed by kshitij
parent 81d4619707
commit 653a95f9fa
2 changed files with 43 additions and 6 deletions

View file

@ -3304,6 +3304,10 @@ class SessionDB:
if not title:
return None
# Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError at
# UTF-8 encode time) — scrub them like every other write path here.
title = _sanitize_surrogates(title)
# Remove ASCII control characters (0x00-0x1F, 0x7F) but keep
# whitespace chars (\t=0x09, \n=0x0A, \r=0x0D) so they can be
# normalized to spaces by the whitespace collapsing step below
@ -4181,7 +4185,10 @@ class SessionDB:
``api_content`` is the exact content string sent to the API for this
message when it differs from ``content`` (ephemeral memory/plugin
injections, persist overrides). It is a byte-fidelity sidecar for
prompt-cache-stable replay stored verbatim, never sanitized.
prompt-cache-stable replay stored as sent, except lone surrogates
(which sqlite3 cannot bind and which the conversation loop scrubs
from every outgoing payload anyway, so the scrubbed form IS the
wire bytes).
"""
# Serialize structured fields to JSON before entering the write txn
reasoning_details_json = (
@ -4229,7 +4236,7 @@ class SessionDB:
stored_content,
tool_call_id,
tool_calls_json,
tool_name,
_scrub_surrogates(tool_name),
effect_disposition,
message_timestamp,
token_count,
@ -4242,7 +4249,7 @@ class SessionDB:
platform_message_id,
1 if observed else 0,
1,
api_content if isinstance(api_content, str) else None,
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
),
)
msg_id = cursor.lastrowid
@ -4325,7 +4332,7 @@ class SessionDB:
self._encode_content(msg.get("content")),
msg.get("tool_call_id"),
tool_calls_json,
msg.get("tool_name"),
_scrub_surrogates(msg.get("tool_name")),
msg.get("effect_disposition"),
message_timestamp,
msg.get("token_count"),
@ -4338,7 +4345,7 @@ class SessionDB:
platform_msg_id,
1 if msg.get("observed") else 0,
1,
api_content if isinstance(api_content, str) else None,
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
),
)
inserted += 1
@ -4489,7 +4496,7 @@ class SessionDB:
"WHERE session_id = ? AND role = 'user' AND active = 1 "
"ORDER BY id DESC LIMIT 1"
") AND content IS ?",
(api_content, session_id, encoded),
(_scrub_surrogates(api_content), session_id, encoded),
)
return cursor.rowcount

View file

@ -6278,3 +6278,33 @@ class TestLoneSurrogatePersistence:
benign = "Ünïcödé ok — 日本語 🎉 emoji fine"
db.append_message("s1", "assistant", benign)
assert db.get_messages("s1")[0]["content"] == benign
# -- sibling raw-str bind sites (follow-up widening of the same bug class)
def test_append_message_survives_lone_surrogate_api_content(self, db):
db.create_session("s1", source="cli")
db.append_message("s1", "user", "clean", api_content=self.DIRTY)
assert db.get_messages("s1")[0]["api_content"] == "scraped \ufffd price"
def test_append_message_survives_lone_surrogate_tool_name(self, db):
db.create_session("s1", source="cli")
db.append_message("s1", "tool", "ok", tool_name="web\ud835search")
assert len(db.get_messages("s1")) == 1
def test_replace_messages_survives_lone_surrogate_api_content(self, db):
db.create_session("s1", source="cli")
db.replace_messages(
"s1", [{"role": "user", "content": "u1", "api_content": self.DIRTY}]
)
assert db.get_messages("s1")[0]["api_content"] == "scraped \ufffd price"
def test_set_latest_user_api_content_survives_lone_surrogate(self, db):
db.create_session("s1", source="cli")
db.append_message("s1", "user", "turn text")
assert db.set_latest_user_api_content("s1", "turn text", self.DIRTY) == 1
def test_session_title_survives_lone_surrogate(self, db):
db.create_session("s1", source="cli")
assert db.set_session_title("s1", "title \ud835 bad") is True
assert db.get_session("s1")["title"] == "title \ufffd bad"