fix(state): widen surrogate scrub to remaining raw-str bind sites
Follow-up to the cherry-picked fix: the same UnicodeEncodeError bind failure was live at sibling sites the PR didn't cover — - api_content sidecar (append_message, _insert_message_rows, set_latest_user_api_content): bound raw; a surrogate in the composed api_content aborted the whole row/UPDATE. Scrubbing is wire-accurate: the conversation loop already scrubs every outgoing payload, so the scrubbed form IS what was sent. - tool_name (both INSERT sites): raw bind. - sanitize_title: session titles from LLM title generation or /title could carry surrogates; scrub before validation. E2E-verified each site raised UnicodeEncodeError on main and persists after this commit. Tests added for all five paths.
This commit is contained in:
parent
81d4619707
commit
653a95f9fa
2 changed files with 43 additions and 6 deletions
|
|
@ -3304,6 +3304,10 @@ class SessionDB:
|
|||
if not title:
|
||||
return None
|
||||
|
||||
# Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError at
|
||||
# UTF-8 encode time) — scrub them like every other write path here.
|
||||
title = _sanitize_surrogates(title)
|
||||
|
||||
# Remove ASCII control characters (0x00-0x1F, 0x7F) but keep
|
||||
# whitespace chars (\t=0x09, \n=0x0A, \r=0x0D) so they can be
|
||||
# normalized to spaces by the whitespace collapsing step below
|
||||
|
|
@ -4181,7 +4185,10 @@ class SessionDB:
|
|||
``api_content`` is the exact content string sent to the API for this
|
||||
message when it differs from ``content`` (ephemeral memory/plugin
|
||||
injections, persist overrides). It is a byte-fidelity sidecar for
|
||||
prompt-cache-stable replay — stored verbatim, never sanitized.
|
||||
prompt-cache-stable replay — stored as sent, except lone surrogates
|
||||
(which sqlite3 cannot bind and which the conversation loop scrubs
|
||||
from every outgoing payload anyway, so the scrubbed form IS the
|
||||
wire bytes).
|
||||
"""
|
||||
# Serialize structured fields to JSON before entering the write txn
|
||||
reasoning_details_json = (
|
||||
|
|
@ -4229,7 +4236,7 @@ class SessionDB:
|
|||
stored_content,
|
||||
tool_call_id,
|
||||
tool_calls_json,
|
||||
tool_name,
|
||||
_scrub_surrogates(tool_name),
|
||||
effect_disposition,
|
||||
message_timestamp,
|
||||
token_count,
|
||||
|
|
@ -4242,7 +4249,7 @@ class SessionDB:
|
|||
platform_message_id,
|
||||
1 if observed else 0,
|
||||
1,
|
||||
api_content if isinstance(api_content, str) else None,
|
||||
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
|
||||
),
|
||||
)
|
||||
msg_id = cursor.lastrowid
|
||||
|
|
@ -4325,7 +4332,7 @@ class SessionDB:
|
|||
self._encode_content(msg.get("content")),
|
||||
msg.get("tool_call_id"),
|
||||
tool_calls_json,
|
||||
msg.get("tool_name"),
|
||||
_scrub_surrogates(msg.get("tool_name")),
|
||||
msg.get("effect_disposition"),
|
||||
message_timestamp,
|
||||
msg.get("token_count"),
|
||||
|
|
@ -4338,7 +4345,7 @@ class SessionDB:
|
|||
platform_msg_id,
|
||||
1 if msg.get("observed") else 0,
|
||||
1,
|
||||
api_content if isinstance(api_content, str) else None,
|
||||
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
|
||||
),
|
||||
)
|
||||
inserted += 1
|
||||
|
|
@ -4489,7 +4496,7 @@ class SessionDB:
|
|||
"WHERE session_id = ? AND role = 'user' AND active = 1 "
|
||||
"ORDER BY id DESC LIMIT 1"
|
||||
") AND content IS ?",
|
||||
(api_content, session_id, encoded),
|
||||
(_scrub_surrogates(api_content), session_id, encoded),
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
|
|
|
|||
|
|
@ -6278,3 +6278,33 @@ class TestLoneSurrogatePersistence:
|
|||
benign = "Ünïcödé ok — 日本語 🎉 emoji fine"
|
||||
db.append_message("s1", "assistant", benign)
|
||||
assert db.get_messages("s1")[0]["content"] == benign
|
||||
|
||||
# -- sibling raw-str bind sites (follow-up widening of the same bug class)
|
||||
|
||||
def test_append_message_survives_lone_surrogate_api_content(self, db):
|
||||
db.create_session("s1", source="cli")
|
||||
db.append_message("s1", "user", "clean", api_content=self.DIRTY)
|
||||
assert db.get_messages("s1")[0]["api_content"] == "scraped \ufffd price"
|
||||
|
||||
def test_append_message_survives_lone_surrogate_tool_name(self, db):
|
||||
db.create_session("s1", source="cli")
|
||||
db.append_message("s1", "tool", "ok", tool_name="web\ud835search")
|
||||
assert len(db.get_messages("s1")) == 1
|
||||
|
||||
def test_replace_messages_survives_lone_surrogate_api_content(self, db):
|
||||
db.create_session("s1", source="cli")
|
||||
db.replace_messages(
|
||||
"s1", [{"role": "user", "content": "u1", "api_content": self.DIRTY}]
|
||||
)
|
||||
assert db.get_messages("s1")[0]["api_content"] == "scraped \ufffd price"
|
||||
|
||||
def test_set_latest_user_api_content_survives_lone_surrogate(self, db):
|
||||
db.create_session("s1", source="cli")
|
||||
db.append_message("s1", "user", "turn text")
|
||||
assert db.set_latest_user_api_content("s1", "turn text", self.DIRTY) == 1
|
||||
|
||||
def test_session_title_survives_lone_surrogate(self, db):
|
||||
db.create_session("s1", source="cli")
|
||||
assert db.set_session_title("s1", "title \ud835 bad") is True
|
||||
assert db.get_session("s1")["title"] == "title \ufffd bad"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue