package sync import ( "strings" "testing" "unicode/utf8" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.kenn.io/agentsview/internal/db" ) func TestSanitizeMessage(t *testing.T) { tests := []struct { name string in db.Message wantRole string wantContent string wantModel string wantCtx int wantOut int wantTS string wantStats validationStats }{ { name: "clean message untouched", in: db.Message{ Role: "assistant", Content: "hello world\nsecond line", Model: "claude-opus-4", ContextTokens: 1000, OutputTokens: 50, Timestamp: "2026-06-20T10:00:00Z", }, wantRole: "assistant", wantContent: "hello world\nsecond line", wantModel: "claude-opus-4", wantCtx: 1000, wantOut: 50, wantTS: "2026-06-20T10:00:00Z", }, { name: "C1 control rune in model stripped", in: db.Message{ Role: "assistant", Model: "claude\u0085-opus", // U+0085 NEL is a C1 control }, wantRole: "assistant", wantModel: "claude-opus", wantStats: validationStats{ControlCharsStripped: 1}, }, { name: "ESC and BEL terminal escape in content stripped", in: db.Message{ Role: "assistant", Content: "before\x1b]0;title\x07after", }, wantRole: "assistant", wantContent: "before]0;titleafter", wantStats: validationStats{ControlCharsStripped: 1}, }, { name: "tab newline cr preserved in content", in: db.Message{ Role: "user", Content: "a\tb\nc\rd", }, wantRole: "user", wantContent: "a\tb\nc\rd", }, { name: "5MB printable model clamped", in: db.Message{ Role: "assistant", Model: strings.Repeat("a", 5_000_000), }, wantRole: "assistant", wantModel: strings.Repeat("a", maxModelLen), wantStats: validationStats{ModelClamped: 1}, }, { name: "dirty model prefix stripped before clamp", in: db.Message{ Role: "assistant", Model: strings.Repeat("\x00", maxModelLen+16) + "claude-opus-4", }, wantRole: "assistant", wantModel: "claude-opus-4", wantStats: validationStats{ControlCharsStripped: 1}, }, { name: "out-of-enum role coerced to blank", in: db.Message{ Role: "wizard", Content: "x", }, wantRole: "", wantContent: "x", wantStats: validationStats{RoleCoerced: 1}, }, { name: "known system and tool roles preserved", in: db.Message{ Role: "system", }, wantRole: "system", }, { name: "over-bound token counts clamped", in: db.Message{ Role: "assistant", ContextTokens: 9_000_000, OutputTokens: 3_000_000, }, wantRole: "assistant", wantCtx: maxPlausibleTokens, wantOut: maxPlausibleTokens, wantStats: validationStats{TokensClamped: 2}, }, { name: "negative token count floored", in: db.Message{ Role: "assistant", OutputTokens: -5, }, wantRole: "assistant", wantOut: 0, wantStats: validationStats{TokensClamped: 1}, }, { name: "out-of-window timestamp blanked", in: db.Message{ Role: "assistant", Timestamp: "1850-01-01T00:00:00Z", }, wantRole: "assistant", wantTS: "", wantStats: validationStats{TimestampsBlanked: 1}, }, { name: "far-future timestamp blanked", in: db.Message{ Role: "assistant", Timestamp: "2999-01-01T00:00:00Z", }, wantRole: "assistant", wantTS: "", wantStats: validationStats{TimestampsBlanked: 1}, }, { name: "unparseable timestamp left as-is", in: db.Message{ Role: "assistant", Timestamp: "not-a-timestamp", }, wantRole: "assistant", wantTS: "not-a-timestamp", }, { name: "empty timestamp untouched", in: db.Message{ Role: "user", Timestamp: "", }, wantRole: "user", wantTS: "", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { m := tc.in stats := sanitizeMessage(&m) assert.Equal(t, tc.wantRole, m.Role, "role") assert.Equal(t, tc.wantContent, m.Content, "content") assert.Equal(t, tc.wantModel, m.Model, "model") assert.Equal(t, tc.wantCtx, m.ContextTokens, "context tokens") assert.Equal(t, tc.wantOut, m.OutputTokens, "output tokens") assert.Equal(t, tc.wantTS, m.Timestamp, "timestamp") assert.Equal(t, tc.wantStats, stats, "stats") }) } } // TestSanitizeMessageRecomputesContentLength verifies that ContentLength // is reduced by the stripped-byte delta. When a parser set it from the // raw content length, after control runes are stripped it must match the // actually-stored byte length, not the original raw length. func TestSanitizeMessageRecomputesContentLength(t *testing.T) { raw := "before\x1b]0;title\x07after" m := db.Message{ Role: "assistant", Content: raw, ContentLength: len(raw), // parser-set raw length } require.Greater(t, len(raw), len("before]0;titleafter"), "raw must be longer than sanitized for this test to matter") stats := sanitizeMessage(&m) assert.Equal(t, "before]0;titleafter", m.Content) assert.Equal(t, len(m.Content), m.ContentLength, "ContentLength must match sanitized content byte length") assert.Less(t, m.ContentLength, len(raw), "ContentLength must drop below the raw length after stripping") assert.Equal(t, 1, stats.ControlCharsStripped) // Idempotent: a second pass strips nothing and leaves length unchanged. wantLen := m.ContentLength second := sanitizeMessage(&m) assert.Equal(t, validationStats{}, second, "second pass must be a no-op") assert.Equal(t, wantLen, m.ContentLength, "length stable on re-run") } // TestSanitizeMessageContentLengthDelta verifies the ContentLength // adjustment is a delta, not an overwrite: a parser-set semantic length // that differs from len(Content) is preserved when nothing is stripped, // and reduced by exactly the removed-byte count (not overwritten to // len(Content)) when control runes are stripped. func TestSanitizeMessageContentLengthDelta(t *testing.T) { t.Run("semantic length preserved when nothing stripped", func(t *testing.T) { content := "clean visible content" // Model a thinking/reasoning-inclusive length that intentionally // exceeds len(Content) by 100 bytes. semantic := len(content) + 100 m := db.Message{ Role: "assistant", Content: content, ContentLength: semantic, } stats := sanitizeMessage(&m) assert.Equal(t, content, m.Content, "clean content untouched") assert.Equal(t, 0, stats.ControlCharsStripped, "nothing should be stripped") assert.Equal(t, semantic, m.ContentLength, "semantic ContentLength must be left untouched when nothing is stripped") }) t.Run("tool-only empty content with nonzero length preserved", func(t *testing.T) { // Tool-only message: empty display Content but a nonzero work // length set by the parser. m := db.Message{ Role: "assistant", Content: "", ContentLength: 4096, } stats := sanitizeMessage(&m) assert.Equal(t, "", m.Content) assert.Equal(t, 0, stats.ControlCharsStripped, "nothing should be stripped") assert.Equal(t, 4096, m.ContentLength, "tool-only ContentLength must not be overwritten to len(Content)=0") }) t.Run("reduced by exactly the removed-byte delta", func(t *testing.T) { raw := "before\x1b]0;title\x07after" sanitized := "before]0;titleafter" removed := len(raw) - len(sanitized) require.Greater(t, removed, 0, "this case requires bytes to be stripped") // Semantic length intentionally larger than len(Content). semantic := len(raw) + 100 m := db.Message{ Role: "assistant", Content: raw, ContentLength: semantic, } stats := sanitizeMessage(&m) assert.Equal(t, sanitized, m.Content) assert.Equal(t, 1, stats.ControlCharsStripped) assert.Equal(t, semantic-removed, m.ContentLength, "ContentLength must drop by exactly the removed-byte count, not be overwritten to len(Content)") assert.NotEqual(t, len(m.Content), m.ContentLength, "delta adjustment must not collapse the semantic length to len(Content)") }) t.Run("thinking-inclusive length reduced by thinking delta", func(t *testing.T) { content := "visible answer" rawThinking := "think\x1b]0;title\x07more" sanitizedThinking := "think]0;titlemore" removed := len(rawThinking) - len(sanitizedThinking) require.Greater(t, removed, 0, "this case requires thinking bytes to be stripped") m := db.Message{ Role: "assistant", Content: content, ThinkingText: rawThinking, ContentLength: len(content) + len(rawThinking), } stats := sanitizeMessage(&m) assert.Equal(t, content, m.Content) assert.Equal(t, sanitizedThinking, m.ThinkingText) assert.Equal(t, 1, stats.ControlCharsStripped) assert.Equal(t, len(content)+len(sanitizedThinking), m.ContentLength, "ContentLength must drop by stripped thinking bytes when thinking contributes to the parser semantic length") assert.Equal(t, len(content)+len(rawThinking)-removed, m.ContentLength, "length adjustment must use the same removed-byte delta as content stripping") }) } func TestSanitizeMessageStripsNULFromResultContent(t *testing.T) { inputRaw := "{\"cmd\":\"ls\x00 -la\"}" inputClean := "{\"cmd\":\"ls -la\"}" resultRaw := "tool\x00result" resultClean := "toolresult" eventRaw := "event\x00content" eventClean := "eventcontent" m := db.Message{ Role: "assistant", ToolCalls: []db.ToolCall{{ ToolUseID: "tu1", ToolName: "Bash", Category: "Bash", InputJSON: inputRaw, ResultContent: resultRaw, ResultContentLength: len(resultRaw), ResultEvents: []db.ToolResultEvent{{ Source: "wait_output", Status: "completed", Content: eventRaw, ContentLength: len(eventRaw), }}, }}, } stats := sanitizeMessage(&m) require.Len(t, m.ToolCalls, 1) tc := m.ToolCalls[0] assert.Equal(t, inputClean, tc.InputJSON) assert.Equal(t, resultClean, tc.ResultContent) assert.Equal(t, len(resultClean), tc.ResultContentLength) require.Len(t, tc.ResultEvents, 1) assert.Equal(t, eventClean, tc.ResultEvents[0].Content) assert.Equal(t, len(eventClean), tc.ResultEvents[0].ContentLength) assert.Equal(t, 3, stats.ControlCharsStripped) second := sanitizeMessage(&m) assert.Equal(t, validationStats{}, second, "second pass must be a no-op") } func TestSanitizeUsageEvent(t *testing.T) { ev := db.UsageEvent{ Source: "api\x1bcall", Model: strings.Repeat("m", 200), InputTokens: 5_000_000, OutputTokens: 10, CacheCreationInputTokens: -1, CacheReadInputTokens: 0, ReasoningTokens: maxPlausibleTokens + 1, OccurredAt: "1700-01-01T00:00:00Z", CostStatus: "ok", } stats := sanitizeUsageEvent(&ev) assert.Equal(t, "apicall", ev.Source) assert.Equal(t, strings.Repeat("m", maxModelLen), ev.Model) assert.Equal(t, maxPlausibleTokens, ev.InputTokens) assert.Equal(t, 10, ev.OutputTokens) assert.Equal(t, 0, ev.CacheCreationInputTokens) assert.Equal(t, maxPlausibleTokens, ev.ReasoningTokens) assert.Equal(t, "", ev.OccurredAt) assert.Equal(t, "ok", ev.CostStatus) assert.Equal(t, 1, stats.ControlCharsStripped) assert.Equal(t, 1, stats.ModelClamped) // InputTokens, CacheCreationInputTokens (negative), ReasoningTokens. assert.Equal(t, 3, stats.TokensClamped) assert.Equal(t, 1, stats.TimestampsBlanked) } func TestSanitizeUsageEventStripsDirtyModelPrefixBeforeClamp(t *testing.T) { ev := db.UsageEvent{ Source: "api", Model: strings.Repeat("\x00", maxModelLen+16) + "claude-opus-4", } stats := sanitizeUsageEvent(&ev) assert.Equal(t, "claude-opus-4", ev.Model) assert.Equal(t, 1, stats.ControlCharsStripped) assert.Zero(t, stats.ModelClamped) } func TestSanitizeSession(t *testing.T) { first := "hi\x07there" name := "clean name" farPast := "1500-01-01T00:00:00Z" good := "2026-06-20T10:00:00Z" s := db.Session{ Project: "proj\x1bx", Machine: "host", Cwd: "/home/u/dev", FirstMessage: &first, SessionName: &name, StartedAt: &farPast, EndedAt: &good, } stats := sanitizeSession(&s) assert.Equal(t, "projx", s.Project) assert.Equal(t, "host", s.Machine) require.NotNil(t, s.FirstMessage) assert.Equal(t, "hithere", *s.FirstMessage) require.NotNil(t, s.SessionName) assert.Equal(t, "clean name", *s.SessionName) assert.Nil(t, s.StartedAt, "absurd started_at blanked to nil") require.NotNil(t, s.EndedAt) assert.Equal(t, good, *s.EndedAt) assert.Equal(t, 2, stats.ControlCharsStripped) // project + first message assert.Equal(t, 1, stats.TimestampsBlanked) } func TestValidateAndSanitizeAggregatesStats(t *testing.T) { s := db.Session{Project: "p\x1bq"} msgs := []db.Message{ {Role: "bogus", Content: "c\x07"}, {Role: "assistant", Model: strings.Repeat("z", 500)}, } events := []db.UsageEvent{ {Source: "s", InputTokens: -3}, } stats := validateAndSanitize(&s, msgs, events) // project + first message content control strips. assert.Equal(t, 2, stats.ControlCharsStripped) assert.Equal(t, 1, stats.RoleCoerced) assert.Equal(t, 1, stats.ModelClamped) assert.Equal(t, 1, stats.TokensClamped) assert.Equal(t, "", msgs[0].Role) assert.Equal(t, "c", msgs[0].Content) assert.Equal(t, strings.Repeat("z", maxModelLen), msgs[1].Model) assert.Equal(t, 0, events[0].InputTokens) } func TestValidateAndSanitizeNilArgs(t *testing.T) { // All-nil call must not panic and must report no fixes. stats := validateAndSanitize(nil, nil, nil) assert.Equal(t, validationStats{}, stats) } // TestValidateAndSanitizeIdempotent verifies the critical invariant: // running the pass over its own output yields no further fixes and an // identical result. This is what keeps fingerprints stable across pushes. func TestValidateAndSanitizeIdempotent(t *testing.T) { s := db.Session{ Project: "p\x1bq", Cwd: "/x", FirstMessage: new("first\x07line"), StartedAt: new("1500-01-01T00:00:00Z"), EndedAt: new("2026-06-20T10:00:00Z"), } msgs := []db.Message{ { Role: "wizard", Content: "esc\x1b]0;t\x07tail\nkeep\ttabs", ThinkingText: "think\u0090ing", // U+0090 is a C1 control Model: strings.Repeat("m", 5_000_000), ContextTokens: 9_000_000, OutputTokens: -10, Timestamp: "2999-01-01T00:00:00Z", SourceUUID: "uuid\x00val", ToolCalls: []db.ToolCall{{ ToolUseID: "tu1", ToolName: "Bash", Category: "Bash", ResultContent: "tool\x00result", ResultContentLength: len("tool\x00result"), ResultEvents: []db.ToolResultEvent{{ Source: "wait_output", Status: "completed", Content: "event\x00content", ContentLength: len("event\x00content"), }}, }}, }, { Role: "assistant", Content: "clean", Timestamp: "2026-06-20T11:00:00Z", }, } events := []db.UsageEvent{ { Source: "api\x1bx", Model: strings.Repeat("z", 999), InputTokens: 5_000_000, OccurredAt: "1700-01-01T00:00:00Z", ReasoningTokens: -1, }, } // First pass. first := validateAndSanitize(&s, msgs, events) assert.Positive(t, first.ControlCharsStripped) assert.Positive(t, first.ModelClamped) assert.Positive(t, first.TokensClamped) assert.Positive(t, first.RoleCoerced) assert.Positive(t, first.TimestampsBlanked) // Snapshot the cleaned values. sAfter := s msgsAfter := append([]db.Message(nil), msgs...) eventsAfter := append([]db.UsageEvent(nil), events...) // Second pass over the already-clean output: no further fixes. second := validateAndSanitize(&s, msgs, events) assert.Equal(t, validationStats{}, second, "second pass must be a no-op") // Values must be byte-identical after the second pass. assert.Equal(t, sAfter, s) assert.Equal(t, msgsAfter, msgs) assert.Equal(t, eventsAfter, events) } // TestSanitizeUTF8Idempotent guards the shared seam directly: the write // path stores SanitizeUTF8(x), and the fingerprint/readback path applies // SanitizeUTF8 again, so it must be a fixed point. func TestSanitizeUTF8Idempotent(t *testing.T) { inputs := []string{ "plain", "keep\n\t\rwhitespace", "esc\x1b]0;title\x07bell", "c1\u0085\u0090\u009f", "nul\x00byte", "bad\xe2utf8", "emoji \U0001F600 ok", } for _, in := range inputs { once := db.SanitizeUTF8(in) twice := db.SanitizeUTF8(once) assert.Equal(t, once, twice, "SanitizeUTF8 must be idempotent for %q", in) assert.True(t, utf8.ValidString(once), "result must be valid UTF-8 for %q", in) } } func TestClampModelRuneBoundary(t *testing.T) { // A multibyte rune straddling the cap must not be split, so the // result stays valid UTF-8 (and re-sanitizing is a no-op). m := strings.Repeat("a", maxModelLen-1) + "é" + "tail" changed := clampModel(&m) assert.True(t, changed) assert.True(t, utf8.ValidString(m), "clamped model must be valid UTF-8") assert.LessOrEqual(t, len(m), maxModelLen) assert.Equal(t, m, db.SanitizeUTF8(m), "clamped model must survive re-sanitization") }