Files
mempalace--mempalace/tests/test_convo_miner_unit.py
wehub-resource-sync c3749daf48
Tests / test-linux (3.13) (push) Failing after 0s
Tests / test-linux (3.11) (push) Failing after 1s
Tests / lint (push) Failing after 0s
Tests / test-linux (3.9) (push) Failing after 1s
Docker / build (push) Failing after 1s
Docker / build-gpu (push) Failing after 2s
Tests / test-windows (push) Has been cancelled
Tests / test-macos (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:03:03 +08:00

622 lines
26 KiB
Python

"""Unit tests for convo_miner pure functions (no chromadb needed)."""
import contextlib
import sys
import pytest
from mempalace.convo_miner import (
CHUNK_SIZE,
_emit_bounded,
_extract_authored_at,
_file_chunks_locked,
chunk_exchanges,
detect_convo_room,
scan_convos,
)
class TestChunkExchanges:
def test_exchange_chunking(self):
content = (
"> What is memory?\n"
"Memory is persistence of information over time.\n\n"
"> Why does it matter?\n"
"It enables continuity across sessions and conversations.\n\n"
"> How do we build it?\n"
"With structured storage and retrieval mechanisms.\n"
)
chunks = chunk_exchanges(content)
assert len(chunks) >= 2
assert all("content" in c and "chunk_index" in c for c in chunks)
def test_paragraph_fallback(self):
"""Content without '>' lines falls back to paragraph chunking."""
content = (
"This is a long paragraph about memory systems. " * 10 + "\n\n"
"This is another paragraph about storage. " * 10 + "\n\n"
"And a third paragraph about retrieval. " * 10
)
chunks = chunk_exchanges(content)
assert len(chunks) >= 2
def test_paragraph_line_group_fallback(self):
"""Long content with no paragraph breaks chunks by line groups.
Each emitted drawer must respect CHUNK_SIZE. Before #1534 the
fallback chunker emitted one drawer per 25-line group without
a size cap, so a 25-line group of long lines produced an
oversized drawer that crashed embedding upsert.
"""
lines = [f"Line {i}: some content that is meaningful" for i in range(60)]
content = "\n".join(lines)
chunks = chunk_exchanges(content)
assert len(chunks) >= 1
max_len = max(len(c["content"]) for c in chunks)
assert max_len <= CHUNK_SIZE, f"oversized chunk: max_len={max_len}"
def test_line_group_fallback_drops_sub_min_trailing_group(self):
"""A trailing line-group whose stripped length is at or below
MIN_CHUNK_SIZE must be dropped, not emitted as a tiny drawer."""
lines = [f"Line {i}" for i in range(51)]
content = "\n".join(lines)
chunks = chunk_exchanges(content)
from mempalace.convo_miner import MIN_CHUNK_SIZE
assert len(chunks) == 2, (
f"expected 2 drawers (groups 0-24 and 25-49); got {len(chunks)}; "
f"the single-line tail group should drop below MIN_CHUNK_SIZE={MIN_CHUNK_SIZE}"
)
def test_empty_content(self):
chunks = chunk_exchanges("")
assert chunks == []
def test_short_content_skipped(self):
chunks = chunk_exchanges("> hi\nbye")
# Too short to produce chunks (below MIN_CHUNK_SIZE)
assert isinstance(chunks, list)
def test_chunk_size_zero_raises_valueerror(self):
"""Reject chunk_size == 0 explicitly.
Without this guard, `_chunk_by_exchange` enters an infinite loop:
content[:0] is empty, content[0:] is the whole string, and the
remainder never shrinks.
"""
content = (
"> What is memory?\nMemory is persistence.\n\n" * 4 # force the split branch
)
with pytest.raises(ValueError, match="chunk_size must be > 0"):
chunk_exchanges(content, chunk_size=0)
def test_chunk_size_negative_raises_valueerror(self):
"""Reject chunk_size < 0. Negative slicing would also loop forever
(content[:-1] → all but last, remainder[-1:] → last char repeated)."""
content = "> hi\nsome response text here that is long enough to chunk\n\n" * 4
with pytest.raises(ValueError, match="chunk_size must be > 0"):
chunk_exchanges(content, chunk_size=-10)
def test_min_chunk_size_negative_raises_valueerror(self):
"""Reject min_chunk_size < 0. A negative threshold silently
breaks the `if len(part.strip()) > min_chunk_size` gate — every
chunk including empty ones gets appended."""
with pytest.raises(ValueError, match="min_chunk_size must be >= 0"):
chunk_exchanges("> hi\nbye", min_chunk_size=-1)
def test_min_chunk_size_zero_allowed(self):
"""min_chunk_size == 0 is legal — means 'accept any non-empty chunk'."""
content = "> What is memory?\nMemory is persistence of information.\n" * 3
chunks = chunk_exchanges(content, min_chunk_size=0)
assert isinstance(chunks, list)
def test_long_ai_response_not_truncated(self):
"""AI responses longer than 8 lines must be stored in full (verbatim principle)."""
lines = [f"Step {i}: important detail that must be stored" for i in range(1, 14)]
content = "> How do I implement authentication?\n" + "\n".join(lines)
chunks = chunk_exchanges(content)
assert len(chunks) >= 1
stored = chunks[0]["content"]
# All 13 lines must be present — none silently dropped
for i in range(1, 14):
assert f"Step {i}:" in stored, f"Step {i} was truncated and not stored"
def test_paragraph_loop_enforces_chunk_size(self):
"""A paragraph longer than CHUNK_SIZE must split into multiple
bounded drawers. Regression for #1534: the paragraph loop in
``_chunk_by_paragraph`` used to append each paragraph as a
single drawer regardless of size, producing one giant chunk
that crashed embedding upsert with
``RuntimeError: Invalid buffer size: ... GiB``.
"""
big_para = "x" * 5000
tail = "small paragraph tail of meaningful length"
content = big_para + "\n\n" + tail
chunks = chunk_exchanges(content)
max_len = max(len(c["content"]) for c in chunks)
assert max_len <= CHUNK_SIZE, f"oversized chunk: max_len={max_len}"
assert len(chunks) > 1, "5000-char content should produce multiple drawers"
assert chunks[-1]["content"] == tail, (
"trailing paragraph must be preserved as the last drawer"
)
def test_custom_chunk_size_propagates_to_paragraph_path(self):
"""User-supplied chunk_size must govern the paragraph chunker, not
only the exchange chunker. Confirms config plumbing reaches both
paths after the #1534 fix.
"""
big_para = "y" * 3000
content = big_para + "\n\ntail paragraph of meaningful length"
chunks = chunk_exchanges(content, chunk_size=400)
max_len = max(len(c["content"]) for c in chunks)
assert max_len <= 400, f"oversized chunk under custom chunk_size=400: {max_len}"
def test_paragraph_loop_no_content_loss(self):
"""Verbatim principle: every char of a single long paragraph lands
in some drawer in order. The slicing helper must not drop or
reorder content."""
content = "a" * 5000
chunks = chunk_exchanges(content)
joined = "".join(c["content"] for c in chunks)
assert joined == content
def test_chunk_exactly_at_size_boundary(self):
"""Content length == CHUNK_SIZE produces exactly one drawer of CHUNK_SIZE."""
content = "z" * CHUNK_SIZE
chunks = chunk_exchanges(content)
assert len(chunks) == 1
assert len(chunks[0]["content"]) == CHUNK_SIZE
def test_chunk_many_multiples_of_size(self):
"""Content length == 8 * CHUNK_SIZE produces exactly 8 drawers, each
of length CHUNK_SIZE."""
content = "w" * (8 * CHUNK_SIZE)
chunks = chunk_exchanges(content)
assert len(chunks) == 8
assert all(len(c["content"]) == CHUNK_SIZE for c in chunks)
def test_paragraph_loop_preserves_slice_order(self):
"""Slices must appear in source order. Guards against a future
regression where the helper reverses, shuffles, or duplicates
slices — the verbatim invariant in CLAUDE.md depends on order
as well as content."""
content = "a" * CHUNK_SIZE + "b" * CHUNK_SIZE + "c" * CHUNK_SIZE
chunks = chunk_exchanges(content)
assert len(chunks) == 3
assert chunks[0]["content"] == "a" * CHUNK_SIZE
assert chunks[1]["content"] == "b" * CHUNK_SIZE
assert chunks[2]["content"] == "c" * CHUNK_SIZE
def test_ai_response_preserves_blank_lines(self):
"""Blank lines inside an AI response must survive ingestion (verbatim principle).
A response with paragraph breaks separates distinct ideas; collapsing the
blank lines loses that boundary and fuses unrelated content.
"""
# Three `>` turns route through _chunk_by_exchange (the exchange-pair path).
content = (
"> explain the architecture\n"
"First paragraph introducing the system.\n"
"\n"
"Second paragraph about the data layer.\n"
"\n"
"Third paragraph about retrieval.\n"
"\n"
"> what about caching?\n"
"Cache lives in memory and is invalidated on write.\n"
"\n"
"> and persistence?\n"
"Persistence lives on disk via SQLite and Chroma.\n"
)
chunks = chunk_exchanges(content)
assert len(chunks) >= 1
stored = "\n".join(c["content"] for c in chunks)
# Paragraph break between the three bodies must survive as `\n\n`.
assert "First paragraph introducing the system.\n\nSecond paragraph" in stored
assert "Second paragraph about the data layer.\n\nThird paragraph" in stored
def test_ai_response_preserves_line_structure(self):
"""Line-oriented content (lists, code fences, tables) must keep newlines.
Joining lines with a single space fuses structurally distinct tokens,
breaks downstream search, and destroys code blocks.
"""
content = (
"> show me the steps\n"
"1. First step\n"
"2. Second step\n"
"3. Third step\n"
"```python\n"
"def hello():\n"
" return 'world'\n"
"```\n"
"\n"
"> what next?\n"
"Run the test suite.\n"
"\n"
"> anything else?\n"
"Ship the feature.\n"
)
chunks = chunk_exchanges(content)
assert len(chunks) >= 1
stored = "\n".join(c["content"] for c in chunks)
# Each list item keeps its own line (not "1. First step 2. Second step").
assert "1. First step\n2. Second step\n3. Third step" in stored
# Code fence survives intact, with indentation preserved.
assert "```python\ndef hello():\n return 'world'\n```" in stored
class TestEmitBounded:
"""Direct unit tests for the chunk-size-enforcement helper."""
def test_emits_no_oversized_chunks(self):
chunks = []
_emit_bounded(chunks, "abc" * 20, chunk_size=10, min_chunk_size=0)
assert all(len(c["content"]) <= 10 for c in chunks)
def test_assigns_sequential_chunk_indices(self):
chunks = []
_emit_bounded(chunks, "x" * 25, chunk_size=10, min_chunk_size=0)
assert [c["chunk_index"] for c in chunks] == [0, 1, 2]
def test_continues_existing_chunk_index(self):
chunks = [{"content": "pre-existing entry", "chunk_index": 0}]
_emit_bounded(chunks, "y" * 5, chunk_size=10, min_chunk_size=0)
assert len(chunks) == 2
assert chunks[1]["chunk_index"] == 1
def test_empty_content_noop(self):
chunks = []
_emit_bounded(chunks, "", chunk_size=10, min_chunk_size=0)
assert chunks == []
def test_small_trailing_slice_preserved(self):
"""Once the whole content passes the floor, every slice is emitted
verbatim so small trailing remainders are not silently dropped.
Regression test for the data-loss class flagged on PR #1538."""
chunks = []
_emit_bounded(chunks, "z" * 23, chunk_size=10, min_chunk_size=5)
assert len(chunks) == 3
assert [len(c["content"]) for c in chunks] == [10, 10, 3]
assert "".join(c["content"] for c in chunks) == "z" * 23
def test_trailing_whitespace_slice_preserved_when_whole_passes(self):
"""When the whole content passes the floor, a trailing
whitespace-only slice is preserved verbatim rather than dropped.
The floor is a noise filter on the WHOLE input, not a per-slice gate."""
chunks = []
_emit_bounded(chunks, "a" * 10 + " " * 10, chunk_size=10, min_chunk_size=5)
assert len(chunks) == 2
assert chunks[0]["content"] == "a" * 10
assert chunks[1]["content"] == " " * 10
def test_whole_content_below_floor_dropped(self):
"""The floor is applied to the stripped whole content. An all-whitespace
input (stripped length 0) or a too-short input is dropped without slicing."""
chunks = []
_emit_bounded(chunks, " " * 100, chunk_size=10, min_chunk_size=5)
_emit_bounded(chunks, "ab", chunk_size=10, min_chunk_size=5)
assert chunks == []
def test_split_805_chars_at_chunk_size_800_preserves_tail(self):
"""805 chars at chunk_size=800 produces a 5-char tail. With the
whole-content floor (not per-slice), the 5-char tail is preserved
verbatim. Directly addresses the data-loss scenario raised on PR #1538."""
chunks = []
_emit_bounded(chunks, "y" * 805, chunk_size=800, min_chunk_size=30)
assert len(chunks) == 2
assert chunks[0]["content"] == "y" * 800
assert chunks[1]["content"] == "y" * 5
assert "".join(c["content"] for c in chunks) == "y" * 805
class TestDetectConvoRoom:
def test_technical_room(self):
content = "Let me debug this python function and fix the code error in the api"
assert detect_convo_room(content) == "technical"
def test_planning_room(self):
content = "We need to plan the roadmap for the next sprint and set milestone deadlines"
assert detect_convo_room(content) == "planning"
def test_architecture_room(self):
content = "The architecture uses a service layer with component interface and module design"
assert detect_convo_room(content) == "architecture"
def test_decisions_room(self):
content = "We decided to switch and migrated to the new framework after we chose it"
assert detect_convo_room(content) == "decisions"
def test_general_fallback(self):
content = "Hello, how are you doing today? The weather is nice."
assert detect_convo_room(content) == "general"
class TestScanConvos:
def test_scan_finds_txt_and_md(self, tmp_path):
(tmp_path / "chat.txt").write_text("hello", encoding="utf-8")
(tmp_path / "notes.md").write_text("world", encoding="utf-8")
(tmp_path / "image.png").write_bytes(b"fake")
files = scan_convos(str(tmp_path))
extensions = {f.suffix for f in files}
assert ".txt" in extensions
assert ".md" in extensions
assert ".png" not in extensions
def test_scan_skips_git_dir(self, tmp_path):
git_dir = tmp_path / ".git"
git_dir.mkdir()
(git_dir / "config.txt").write_text("git stuff", encoding="utf-8")
(tmp_path / "chat.txt").write_text("hello", encoding="utf-8")
files = scan_convos(str(tmp_path))
assert len(files) == 1
def test_scan_skips_meta_json(self, tmp_path):
(tmp_path / "chat.meta.json").write_text("{}", encoding="utf-8")
(tmp_path / "chat.json").write_text("{}", encoding="utf-8")
files = scan_convos(str(tmp_path))
names = [f.name for f in files]
assert "chat.json" in names
assert "chat.meta.json" not in names
def test_scan_empty_dir(self, tmp_path):
files = scan_convos(str(tmp_path))
assert files == []
@pytest.mark.skipif(
sys.platform == "win32",
reason="symlink creation requires elevated privileges on Windows",
)
def test_scan_convos_logs_skipped_symlinks(self, tmp_path, capsys):
real_target = tmp_path / "outside" / "real.jsonl"
real_target.parent.mkdir()
real_target.write_text('{"role":"user","content":"hi"}\n', encoding="utf-8")
link_root = tmp_path / "link_root"
link_root.mkdir()
(link_root / "link.jsonl").symlink_to(real_target)
(link_root / "regular.jsonl").write_text(
'{"role":"user","content":"hello"}\n', encoding="utf-8"
)
files = scan_convos(str(link_root))
names = {f.name for f in files}
assert "link.jsonl" not in names
assert "regular.jsonl" in names
err = capsys.readouterr().err
assert err.count("SKIP:") == 1
assert " SKIP:" in err
assert "link.jsonl" in err
assert "(symlink)" in err
@pytest.mark.skipif(
sys.platform == "win32",
reason="symlink creation requires elevated privileges on Windows",
)
def test_scan_convos_logs_dangling_symlink(self, tmp_path, capsys):
real_target = tmp_path / "outside" / "ghost.jsonl"
real_target.parent.mkdir()
real_target.touch()
link_root = tmp_path / "link_root"
link_root.mkdir()
(link_root / "dangling.jsonl").symlink_to(real_target)
real_target.unlink() # target deleted, link dangles
files = scan_convos(str(link_root))
assert files == []
err = capsys.readouterr().err
assert err.count("SKIP:") == 1
assert "dangling.jsonl" in err
assert "(symlink)" in err
@pytest.mark.skipif(
sys.platform == "win32",
reason="symlink creation requires elevated privileges on Windows",
)
def test_scan_convos_logs_nested_symlink_with_relative_path(self, tmp_path, capsys):
real_target = tmp_path / "outside" / "real.jsonl"
real_target.parent.mkdir()
real_target.write_text('{"x":1}\n', encoding="utf-8")
link_root = tmp_path / "link_root"
subdir = link_root / "deep" / "subdir"
subdir.mkdir(parents=True)
(subdir / "nested.jsonl").symlink_to(real_target)
files = scan_convos(str(link_root))
assert files == []
err = capsys.readouterr().err
# Forward slash even on Windows (as_posix) and full relative path,
# not just the leaf — proves relative_to(convo_path) over .name.
assert "deep/subdir/nested.jsonl" in err
assert "(symlink)" in err
def test_scan_skips_oversized_files(self, tmp_path, capsys, monkeypatch):
import re
import mempalace.convo_miner as convo_mod
monkeypatch.setattr(convo_mod, "MAX_FILE_SIZE", 100)
(tmp_path / "small.txt").write_text("hello " * 5, encoding="utf-8")
(tmp_path / "big.txt").write_text("hello " * 100, encoding="utf-8")
files = scan_convos(str(tmp_path))
names = [f.name for f in files]
assert "small.txt" in names
assert "big.txt" not in names
err = capsys.readouterr().err
# SKIP message goes to stderr, matching the existing
# `SKIP: <rel> (symlink)` line in the same function.
assert "SKIP: big.txt" in err
# Validate the full template so a drop of the MB suffix or a
# regression to bare-substring output trips the test.
assert re.search(r"SKIP: big\.txt \(\d+\.\d+ MB\) exceeds \d+ MB limit", err), err
def test_scan_skips_unreadable_files(self, tmp_path, capsys, monkeypatch):
from pathlib import Path
# .txt is in CONVO_EXTENSIONS so it reaches the size-check gate.
(tmp_path / "readable.txt").write_text("hi", encoding="utf-8")
unreadable = tmp_path / "unreadable.txt"
unreadable.write_text("hi", encoding="utf-8")
real_stat = Path.stat
def selective_stat(self, *args, **kwargs):
# On Py 3.10+, Path.is_symlink() routes through lstat ->
# stat(follow_symlinks=False). Only raise for the follow-symlinks
# call that the actual size-check makes, otherwise the test
# never reaches the size-check arm we want to exercise.
if self.name == "unreadable.txt" and kwargs.get("follow_symlinks", True):
raise PermissionError(13, "Permission denied", str(self))
return real_stat(self, *args, **kwargs)
monkeypatch.setattr(Path, "stat", selective_stat)
files = scan_convos(str(tmp_path))
names = [f.name for f in files]
assert "readable.txt" in names
assert "unreadable.txt" not in names
err = capsys.readouterr().err
assert "SKIP: unreadable.txt" in err
assert "stat error" in err
class TestFileChunksLocked:
def test_uses_bounded_upsert_batches(self, monkeypatch):
import mempalace.convo_miner as convo_miner
class FakeCol:
def __init__(self):
self.batch_sizes = []
def delete(self, *args, **kwargs):
pass
def get(self, ids=None, include=None, **kwargs):
# Pre-mining collision scan probes the collection; empty
# palace under test, so nothing matches.
return {"ids": [], "metadatas": []}
def upsert(self, documents, ids, metadatas):
self.batch_sizes.append(len(documents))
chunks = [{"content": f"chunk {i} " * 20, "chunk_index": i} for i in range(5)]
col = FakeCol()
monkeypatch.setattr(convo_miner, "DRAWER_UPSERT_BATCH_SIZE", 2)
monkeypatch.setattr(
convo_miner, "file_already_mined", lambda collection, source_file, **kwargs: False
)
monkeypatch.setattr(convo_miner, "mine_lock", lambda source_file: contextlib.nullcontext())
monkeypatch.setattr(convo_miner, "_detect_hall_cached", lambda content: "conversations")
drawers, room_counts, skipped = _file_chunks_locked(
col, "chat.txt", chunks, "wing", "general", "agent", "exchange"
)
assert drawers == 5
assert dict(room_counts) == {}
assert skipped is False
assert col.batch_sizes == [2, 2, 1]
def test_populates_entities_metadata(self, monkeypatch):
import mempalace.convo_miner as convo_miner
class FakeCol:
def __init__(self):
self.metas = []
def delete(self, *args, **kwargs):
pass
def get(self, ids=None, include=None, **kwargs):
return {"ids": [], "metadatas": []}
def upsert(self, documents, ids, metadatas):
self.metas.extend(metadatas)
chunks = [
{
"content": "We changed `MemoryStack` in rag/foo.py via do_thing_now().",
"chunk_index": 0,
}
]
col = FakeCol()
monkeypatch.setattr(
convo_miner, "file_already_mined", lambda collection, source_file, **kwargs: False
)
monkeypatch.setattr(convo_miner, "mine_lock", lambda source_file: contextlib.nullcontext())
monkeypatch.setattr(convo_miner, "_detect_hall_cached", lambda content: "conversations")
_file_chunks_locked(col, "chat.txt", chunks, "wing", "general", "agent", "exchange")
entities = col.metas[0]["entities"].split(";")
assert "MemoryStack" in entities
assert "rag/foo.py" in entities
assert "do_thing_now" in entities
class TestExtractAuthoredAt:
"""authored_at = max per-line ``timestamp`` in a transcript (real authored date,
independent of mine time). Both Claude Code and Codex JSONL carry a top-level
ISO-8601 ``timestamp`` per line."""
def test_returns_latest_timestamp(self, tmp_path):
f = tmp_path / "session.jsonl"
f.write_text(
'{"type": "user", "timestamp": "2026-06-21T10:00:00.000Z"}\n'
'{"type": "assistant", "timestamp": "2026-06-23T14:30:00.000Z"}\n'
'{"type": "user", "timestamp": "2026-06-22T09:00:00.000Z"}\n'
)
assert _extract_authored_at(f) == "2026-06-23T14:30:00.000Z"
def test_ignores_lines_without_timestamp(self, tmp_path):
f = tmp_path / "session.jsonl"
f.write_text(
'{"type": "summary", "summary": "x"}\n'
'{"type": "assistant", "timestamp": "2026-06-23T14:30:00.000Z"}\n'
)
assert _extract_authored_at(f) == "2026-06-23T14:30:00.000Z"
def test_tolerates_blank_and_malformed_lines(self, tmp_path):
f = tmp_path / "session.jsonl"
f.write_text(
"\n"
"not json\n"
"[1, 2, 3]\n" # valid JSON, but no .get()
'{"timestamp": "2026-06-25T00:00:00.000Z"}\n'
)
assert _extract_authored_at(f) == "2026-06-25T00:00:00.000Z"
def test_none_for_non_jsonl(self, tmp_path):
f = tmp_path / "notes.md"
f.write_text("# heading\n")
assert _extract_authored_at(f) is None
def test_none_when_no_timestamps(self, tmp_path):
f = tmp_path / "session.jsonl"
f.write_text('{"type": "user", "content": "hi"}\n')
assert _extract_authored_at(f) is None
def test_none_for_missing_file(self, tmp_path):
assert _extract_authored_at(tmp_path / "absent.jsonl") is None
def test_non_string_timestamp_does_not_crash(self, tmp_path):
# A non-string timestamp must be skipped, not raise TypeError on compare.
f = tmp_path / "session.jsonl"
f.write_text(
'{"type": "user", "timestamp": 1234567890}\n'
'{"type": "assistant", "timestamp": {"nested": true}}\n'
'{"type": "user", "timestamp": "2026-06-24T00:00:00.000Z"}\n'
)
assert _extract_authored_at(f) == "2026-06-24T00:00:00.000Z"
def test_only_non_string_timestamps_returns_none(self, tmp_path):
f = tmp_path / "session.jsonl"
f.write_text('{"timestamp": 1}\n{"timestamp": false}\n')
assert _extract_authored_at(f) is None