chore: import upstream snapshot with attribution
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
CI (OpenClaw E2E) / openclaw test (push) Has been cancelled
CI / coverage-report (push) Has been cancelled
CI / test-kubernetes (push) Has been cancelled
CI / should-run-thorough (push) Has been cancelled
CI / test-thorough (cloudwatch-demo) (push) Has been cancelled
CI / test-thorough (flink-ecs) (push) Has been cancelled
CI / test-thorough (upstream-lambda) (push) Has been cancelled
CI / test-thorough (prefect-ecs-fargate) (push) Has been cancelled
Release / build-binaries (zip, opensre.exe, onefile, windows-latest, windows-x64) (push) Has been cancelled
Benchmark image — build + push to ECR (any adapter) / build + push (push) Has been cancelled
CI / quality (ubuntu-latest) (push) Has been cancelled
CI / test (tools-runtime) (push) Has been cancelled
CI / test (e2e-general) (push) Has been cancelled
CI / test (cli-runtime) (push) Has been cancelled
CI / test (e2e-provider-and-openclaw) (push) Has been cancelled
CI / test (integrations-and-misc) (push) Has been cancelled
Release / verify (push) Has been cancelled
Release / build-python-dist (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-15-intel, darwin-x64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, macos-latest, darwin-arm64) (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04, linux-x64) (push) Has been cancelled
Release / publish-release (push) Has been cancelled
Release / publish-main-release (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-checks (no-LLM) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Interactive Shell Live (PR + post-merge) / turn-live shard ${{ matrix.shard_index }} (push) Has been cancelled
Release / prepare (push) Has been cancelled
Release / build-binaries (tar.gz, opensre, onedir, ubuntu-22.04-arm, linux-arm64) (push) Has been cancelled
Synthetic Deterministic Tests / Synthetic offline (deterministic) (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
"""Tests for interactive-shell LLM grounding reference corpora."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Tests for the AGENTS.md grounding helpers used by the interactive shell."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.agent_harness.grounding import agents_md_reference
|
||||
from core.agent_harness.grounding._cache import excerpt
|
||||
from core.agent_harness.grounding.agents_md_reference import (
|
||||
AgentsMdFile,
|
||||
AgentsMdReference,
|
||||
)
|
||||
|
||||
|
||||
def _write(root: Path, relpath: str, content: str) -> None:
|
||||
path = root / relpath
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _seed_agents_md(root: Path) -> None:
|
||||
_write(root, "AGENTS.md", "# Repo map\n\nTop-level guidance.\n")
|
||||
_write(root, "core/llm/AGENTS.md", "# Hosted LLM Runtime\n\nLLM API clients.\n")
|
||||
_write(root, "integrations/llm_cli/AGENTS.md", "# llm_cli\n\nSubprocess CLIs.\n")
|
||||
# Files that MUST be skipped — tests, vcs, caches, virtualenv, vendored deps.
|
||||
_write(root, "tests/AGENTS.md", "should be skipped")
|
||||
_write(root, ".git/AGENTS.md", "should be skipped")
|
||||
_write(root, "__pycache__/AGENTS.md", "should be skipped")
|
||||
_write(root, "node_modules/some-pkg/AGENTS.md", "should be skipped")
|
||||
_write(root, ".venv/lib/python3.13/site-packages/foo/AGENTS.md", "should be skipped")
|
||||
|
||||
|
||||
class TestDiscoverAgentsMdFiles:
|
||||
def test_returns_empty_list_when_root_missing(self, tmp_path: Path) -> None:
|
||||
missing = tmp_path / "no-such-dir"
|
||||
assert AgentsMdReference().discover(missing) == []
|
||||
|
||||
def test_walks_root_and_skips_excluded_dirs(self, tmp_path: Path) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
files = AgentsMdReference().discover(tmp_path)
|
||||
relpaths = {f.relpath for f in files}
|
||||
assert "AGENTS.md" in relpaths
|
||||
assert "core/llm/AGENTS.md" in relpaths
|
||||
assert "integrations/llm_cli/AGENTS.md" in relpaths
|
||||
# None of the skip-dir paths should leak into the index.
|
||||
for r in relpaths:
|
||||
assert not r.startswith("tests/")
|
||||
assert not r.startswith(".git/")
|
||||
assert not r.startswith("__pycache__/")
|
||||
assert not r.startswith("node_modules/")
|
||||
assert not r.startswith(".venv/")
|
||||
|
||||
def test_results_are_sorted_by_relpath(self, tmp_path: Path) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
files = AgentsMdReference().discover(tmp_path)
|
||||
relpaths = [f.relpath for f in files]
|
||||
assert relpaths == sorted(relpaths)
|
||||
|
||||
def test_body_is_verbatim_no_frontmatter_stripping(self, tmp_path: Path) -> None:
|
||||
# Unlike docs_reference, AGENTS.md files are plain Markdown — frontmatter
|
||||
# delimiters (if a contributor adds them) must be preserved verbatim.
|
||||
body = "---\nfoo: bar\n---\n\n# Title\n\nBody.\n"
|
||||
_write(tmp_path, "AGENTS.md", body)
|
||||
files = AgentsMdReference().discover(tmp_path)
|
||||
assert len(files) == 1
|
||||
assert files[0].body == body
|
||||
|
||||
|
||||
class TestBuildAgentsMdReferenceText:
|
||||
def test_returns_empty_when_no_files_present(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
monkeypatch.setattr(agents_md_reference, "REPO_ROOT", tmp_path / "missing")
|
||||
assert AgentsMdReference().build_text() == ""
|
||||
|
||||
def test_concatenates_each_file_with_labelled_header(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
monkeypatch.setattr(agents_md_reference, "REPO_ROOT", tmp_path)
|
||||
text = AgentsMdReference().build_text()
|
||||
# Root file must be disambiguated from per-package files.
|
||||
assert "=== AGENTS.md (root) ===" in text
|
||||
assert "=== core/llm/AGENTS.md ===" in text
|
||||
assert "=== integrations/llm_cli/AGENTS.md ===" in text
|
||||
# Bodies must appear in the concatenated block.
|
||||
assert "Top-level guidance." in text
|
||||
assert "LLM API clients." in text
|
||||
assert "Subprocess CLIs." in text
|
||||
|
||||
def test_caps_total_to_max_chars(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
monkeypatch.setattr(agents_md_reference, "REPO_ROOT", tmp_path)
|
||||
text = AgentsMdReference().build_text(max_chars=100)
|
||||
# Allow a small margin for the truncation marker appended at the cap.
|
||||
assert len(text) <= 200
|
||||
assert "truncated" in text
|
||||
|
||||
|
||||
class TestExcerpt:
|
||||
def test_returns_full_body_when_short(self) -> None:
|
||||
assert excerpt("short body", max_chars=100) == "short body"
|
||||
|
||||
def test_truncates_long_body_at_paragraph_boundary(self) -> None:
|
||||
body = ("paragraph one. " * 10) + "\n\n" + ("paragraph two. " * 10)
|
||||
out = excerpt(body, max_chars=80)
|
||||
assert "truncated" in out
|
||||
# The cut should happen at or before the second paragraph, since the
|
||||
# rfind for "\n\n" before max_chars is the preferred cutoff.
|
||||
assert "paragraph two" not in out
|
||||
|
||||
|
||||
class TestAgentsMdFileDataclass:
|
||||
def test_is_hashable_and_immutable(self) -> None:
|
||||
f = AgentsMdFile(relpath="AGENTS.md", body="hello")
|
||||
assert f in {f}
|
||||
|
||||
|
||||
class TestAgentsMdGroundingCache:
|
||||
def test_cache_maxsize_matches_implementation(self) -> None:
|
||||
stats = AgentsMdReference().stats()
|
||||
assert stats.maxsize == 32
|
||||
|
||||
def test_repeated_discover_hits_parse_cache(self, tmp_path: Path) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
ref = AgentsMdReference()
|
||||
ref.discover(tmp_path)
|
||||
info1 = ref.stats()
|
||||
ref.discover(tmp_path)
|
||||
info2 = ref.stats()
|
||||
assert info2.hits == info1.hits + 1
|
||||
assert info2.misses == info1.misses
|
||||
|
||||
def test_invalidate_resets_stats(self, tmp_path: Path) -> None:
|
||||
_seed_agents_md(tmp_path)
|
||||
ref = AgentsMdReference()
|
||||
ref.discover(tmp_path)
|
||||
ref.discover(tmp_path)
|
||||
assert ref.stats().hits >= 1
|
||||
ref.invalidate()
|
||||
cleared = ref.stats()
|
||||
assert cleared.hits == 0
|
||||
assert cleared.misses == 0
|
||||
assert cleared.currsize == 0
|
||||
|
||||
def test_file_edit_invalidates_and_refreshes_content(self, tmp_path: Path) -> None:
|
||||
ref = AgentsMdReference()
|
||||
_write(tmp_path, "AGENTS.md", "# Repo map\n\nOld content.\n")
|
||||
files1 = ref.discover(tmp_path)
|
||||
assert any("Old content" in f.body for f in files1)
|
||||
|
||||
# Bump mtime well beyond filesystem mtime resolution so the
|
||||
# fingerprint definitely changes on the next discover call.
|
||||
target = tmp_path / "AGENTS.md"
|
||||
target.write_text("# Repo map\n\nNew refreshed content.\n", encoding="utf-8")
|
||||
st = target.stat()
|
||||
os.utime(target, ns=(st.st_atime_ns, st.st_mtime_ns + 2_000_000_000))
|
||||
|
||||
files2 = ref.discover(tmp_path)
|
||||
assert any("New refreshed content" in f.body for f in files2)
|
||||
assert not any("Old content" in f.body for f in files2)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Tests for interactive-shell CLI reference grounding cache."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import click
|
||||
import pytest
|
||||
|
||||
import surfaces.interactive_shell.grounding.cli_reference as cli_reference_module
|
||||
from surfaces.interactive_shell.session.session import Session
|
||||
|
||||
|
||||
def _reference_with_cli() -> cli_reference_module.CliReference:
|
||||
"""Return a :class:`CliReference` wired to the shell's CLI command group.
|
||||
|
||||
CLI catalog assembly lives in ``surfaces/`` (T-05 — see issue #3538); tests
|
||||
inject the group the same way ``ShellPromptContextProvider`` does.
|
||||
"""
|
||||
from surfaces.cli.__main__ import cli
|
||||
|
||||
ref = cli_reference_module.CliReference()
|
||||
ref.set_command_group_provider(lambda: cli)
|
||||
return ref
|
||||
|
||||
|
||||
def test_second_build_is_cache_hit() -> None:
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
s1 = ref.stats()
|
||||
ref.build_text()
|
||||
s2 = ref.stats()
|
||||
assert s2.hits == s1.hits + 1
|
||||
assert s2.misses == s1.misses
|
||||
|
||||
|
||||
def test_cold_build_is_silent(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
from surfaces.cli.__main__ import cli
|
||||
|
||||
text = _reference_with_cli().build_text()
|
||||
captured = capsys.readouterr()
|
||||
first_command = sorted(cli.commands.keys())[0]
|
||||
|
||||
assert captured.out == ""
|
||||
assert captured.err == ""
|
||||
assert "=== opensre --help ===" in text
|
||||
assert f"=== opensre {first_command} --help ===" in text
|
||||
assert f"Usage: opensre {first_command}" in text
|
||||
|
||||
|
||||
def test_invalidate_forces_rebuild_miss() -> None:
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
s1 = ref.stats()
|
||||
assert s1.misses == 1
|
||||
ref.invalidate()
|
||||
assert ref.stats().misses == 0
|
||||
ref.build_text()
|
||||
s2 = ref.stats()
|
||||
assert s2.misses == 1
|
||||
assert s2.cached is True
|
||||
|
||||
|
||||
def test_signature_change_busts_cli_cache(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
ref = _reference_with_cli()
|
||||
monkeypatch.setattr(
|
||||
cli_reference_module,
|
||||
"_current_cli_signature",
|
||||
lambda *_args, **_kwargs: "sig-a",
|
||||
)
|
||||
ref.build_text()
|
||||
monkeypatch.setattr(
|
||||
cli_reference_module,
|
||||
"_current_cli_signature",
|
||||
lambda *_args, **_kwargs: "sig-b",
|
||||
)
|
||||
ref.build_text()
|
||||
stats = ref.stats()
|
||||
assert stats.misses >= 2
|
||||
assert stats.signature == "sig-b"
|
||||
|
||||
|
||||
def test_invalidate_resets_hit_miss_counters() -> None:
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
ref.build_text()
|
||||
assert ref.stats().hits >= 1
|
||||
ref.invalidate()
|
||||
s = ref.stats()
|
||||
assert s.hits == 0
|
||||
assert s.misses == 0
|
||||
|
||||
|
||||
def test_non_cacheable_short_output_skips_store(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
cli_reference_module,
|
||||
"_build_cli_reference_text_uncached",
|
||||
lambda *_args, **_kwargs: "too short",
|
||||
)
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
ref.build_text()
|
||||
stats = ref.stats()
|
||||
assert stats.cached is False
|
||||
assert stats.misses >= 2
|
||||
|
||||
|
||||
def test_non_cacheable_long_without_sentinel_skips_store(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
filler = "x" * 120
|
||||
monkeypatch.setattr(
|
||||
cli_reference_module,
|
||||
"_build_cli_reference_text_uncached",
|
||||
lambda *_args, **_kwargs: filler,
|
||||
)
|
||||
ref = _reference_with_cli()
|
||||
ref.build_text()
|
||||
assert ref.stats().cached is False
|
||||
|
||||
|
||||
def test_reference_without_provider_returns_placeholder() -> None:
|
||||
"""Without a command-group provider the cache emits a placeholder — no surface imports."""
|
||||
ref = cli_reference_module.CliReference()
|
||||
text = ref.build_text()
|
||||
assert "=== opensre --help ===" in text
|
||||
assert "not available in this runtime" in text
|
||||
# Placeholder is intentionally short-lived: it must not populate the cache.
|
||||
assert ref.stats().cached is False
|
||||
|
||||
|
||||
def test_command_group_provider_is_bound_lazily() -> None:
|
||||
"""The provider is invoked only when :meth:`build_text` runs, not at bind time."""
|
||||
calls: list[int] = []
|
||||
|
||||
def _provider() -> click.Command:
|
||||
calls.append(1)
|
||||
group = click.Group("opensre")
|
||||
group.add_command(click.Command("noop"))
|
||||
return group
|
||||
|
||||
ref = cli_reference_module.CliReference()
|
||||
ref.set_command_group_provider(_provider)
|
||||
assert not calls
|
||||
ref.build_text()
|
||||
assert calls == [1]
|
||||
|
||||
|
||||
def test_shell_prompt_context_provider_includes_cli_reference() -> None:
|
||||
provider = cli_reference_module.shell_prompt_context_provider(Session())
|
||||
text = provider.cli_reference()
|
||||
assert "=== opensre --help ===" in text
|
||||
assert "Usage: opensre" in text
|
||||
|
||||
|
||||
def test_shell_prompt_context_provider_reuses_session_cli_cache() -> None:
|
||||
session = Session()
|
||||
first = cli_reference_module.shell_prompt_context_provider(session)
|
||||
second = cli_reference_module.shell_prompt_context_provider(session)
|
||||
first.cli_reference()
|
||||
second.cli_reference()
|
||||
assert second._cli.stats().hits >= 1 # noqa: SLF001 - session-scoped cache reuse
|
||||
@@ -0,0 +1,290 @@
|
||||
"""Tests for the documentation-grounding helpers used by the interactive shell."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from core.agent_harness.grounding import docs_reference
|
||||
from core.agent_harness.grounding._cache import excerpt
|
||||
from core.agent_harness.grounding.docs_reference import (
|
||||
DocPage,
|
||||
DocsReference,
|
||||
_query_tokens,
|
||||
build_docs_index,
|
||||
find_relevant_docs,
|
||||
)
|
||||
|
||||
|
||||
def _write_doc(root: Path, relpath: str, content: str) -> None:
|
||||
path = root / relpath
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _seed_docs(root: Path) -> None:
|
||||
_write_doc(
|
||||
root,
|
||||
"datadog.mdx",
|
||||
'---\ntitle: "Datadog"\n---\n\n'
|
||||
"### Step 1: Create API Key\n\n"
|
||||
"In Datadog, create an API Key under organizational settings.\n\n"
|
||||
"### Step 2: Configure OpenSRE\n\n"
|
||||
"Set DD_API_KEY and DD_APP_KEY in your environment.\n",
|
||||
)
|
||||
_write_doc(
|
||||
root,
|
||||
"deployment.mdx",
|
||||
'---\ntitle: "Deployment"\n---\n\n'
|
||||
"OpenSRE can deploy to Railway or EC2.\n\n"
|
||||
"Use `opensre remote` to connect to a deployed agent.\n",
|
||||
)
|
||||
_write_doc(
|
||||
root,
|
||||
"quickstart.mdx",
|
||||
'---\ntitle: "Quickstart"\n---\n\nInstall OpenSRE and run your first investigation.\n',
|
||||
)
|
||||
_write_doc(
|
||||
root,
|
||||
"tutorials/investigating-task-failures.mdx",
|
||||
"# Investigating task failures\n\n"
|
||||
"Walk through how to investigate a failed task using OpenSRE.\n",
|
||||
)
|
||||
# Asset content under skip dirs MUST be excluded from the index.
|
||||
_write_doc(root, "images/datadog.mdx", "should be skipped")
|
||||
_write_doc(root, "assets/anything.mdx", "should be skipped")
|
||||
|
||||
|
||||
class TestDiscoverDocs:
|
||||
def test_returns_empty_list_when_root_missing(self, tmp_path: Path) -> None:
|
||||
missing = tmp_path / "no-docs"
|
||||
assert DocsReference().discover(missing) == []
|
||||
|
||||
def test_walks_root_and_skips_asset_dirs(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
slugs = {p.slug for p in pages}
|
||||
assert "datadog" in slugs
|
||||
assert "deployment" in slugs
|
||||
assert "quickstart" in slugs
|
||||
assert "investigating-task-failures" in slugs
|
||||
# Anything under images/ or assets/ must be skipped.
|
||||
relpaths = {p.relpath for p in pages}
|
||||
assert all(not r.startswith("images/") for r in relpaths)
|
||||
assert all(not r.startswith("assets/") for r in relpaths)
|
||||
|
||||
def test_extracts_title_from_frontmatter(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
by_slug = {p.slug: p for p in pages}
|
||||
assert by_slug["datadog"].title == "Datadog"
|
||||
assert by_slug["deployment"].title == "Deployment"
|
||||
|
||||
def test_falls_back_to_first_heading_when_no_frontmatter(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
by_slug = {p.slug: p for p in pages}
|
||||
assert by_slug["investigating-task-failures"].title == "Investigating task failures"
|
||||
|
||||
def test_strips_frontmatter_from_body(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
by_slug = {p.slug: p for p in pages}
|
||||
# Frontmatter delimiters and the title line must NOT appear in the body
|
||||
# (they would otherwise leak into the LLM grounding context).
|
||||
assert "title:" not in by_slug["datadog"].body
|
||||
assert by_slug["datadog"].body.lstrip().startswith("###")
|
||||
|
||||
|
||||
class TestQueryTokens:
|
||||
def test_strips_stopwords_and_short_tokens(self) -> None:
|
||||
tokens = _query_tokens("How do I configure Datadog?")
|
||||
# Stopwords are removed, 'datadog' / 'configure' remain.
|
||||
assert "datadog" in tokens
|
||||
assert "configure" in tokens
|
||||
assert "how" not in tokens
|
||||
assert "do" not in tokens
|
||||
assert "i" not in tokens
|
||||
|
||||
def test_drops_opensre_brand_token(self) -> None:
|
||||
# Every doc mentions "opensre" so it would otherwise dominate ranking.
|
||||
tokens = _query_tokens("how do I install opensre")
|
||||
assert "opensre" not in tokens
|
||||
assert "install" in tokens
|
||||
|
||||
def test_keeps_two_letter_tokens(self) -> None:
|
||||
tokens = _query_tokens("how do I tune ai vm sizing")
|
||||
assert "ai" in tokens
|
||||
assert "vm" in tokens
|
||||
|
||||
|
||||
class TestFindRelevantDocs:
|
||||
def test_empty_query_returns_empty(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
# Query with only stopwords should not match anything.
|
||||
assert find_relevant_docs("how do I", pages) == []
|
||||
|
||||
def test_ranks_datadog_page_first_for_datadog_query(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
results = find_relevant_docs("how do I configure Datadog?", pages)
|
||||
assert results, "expected at least one match"
|
||||
assert results[0].slug == "datadog"
|
||||
|
||||
def test_ranks_deployment_page_first_for_deploy_query(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
results = find_relevant_docs("how do I deploy this?", pages)
|
||||
assert results
|
||||
assert results[0].slug == "deployment"
|
||||
|
||||
def test_caps_results_at_top_n(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
results = find_relevant_docs("install configure deploy investigate", pages, top_n=2)
|
||||
assert len(results) <= 2
|
||||
|
||||
def test_nested_page_with_weak_match_is_not_dropped_by_depth(self, tmp_path: Path) -> None:
|
||||
"""A page whose only match is a single body token, nested deep enough
|
||||
that the depth penalty equals or exceeds its raw score, must still
|
||||
surface as a lower-ranked result instead of being excluded entirely.
|
||||
|
||||
Regression: previously the depth penalty was applied unconditionally
|
||||
before the score>0 filter, so a page with raw_score=1 at depth=2
|
||||
scored -1 and was dropped from results.
|
||||
"""
|
||||
# Page nested 2 levels deep whose only match for "masking" is a single
|
||||
# body-token mention. raw_score == 1, depth == 2, so without clamping
|
||||
# the final score would be -1 and the page would be filtered out.
|
||||
_write_doc(
|
||||
tmp_path,
|
||||
"tutorials/advanced/notes.mdx",
|
||||
"# Notes\n\nWe briefly mention masking in this tutorial.\n",
|
||||
)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
results = find_relevant_docs("masking", pages)
|
||||
slugs = [p.slug for p in results]
|
||||
assert "notes" in slugs, (
|
||||
"weak nested match must still surface, not be dropped by depth alone"
|
||||
)
|
||||
|
||||
|
||||
class TestBuildDocsReferenceText:
|
||||
def test_returns_empty_when_no_docs_present(self, tmp_path: Path) -> None:
|
||||
# Point at a non-existent docs root.
|
||||
assert DocsReference().build_text("anything", root=tmp_path / "missing") == ""
|
||||
|
||||
def test_includes_relevant_doc_excerpt_and_index(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
text = DocsReference().build_text("how do I configure Datadog?", root=tmp_path)
|
||||
assert "datadog.mdx" in text
|
||||
assert "API Key" in text
|
||||
# The compact index of all pages must always be appended so the LLM
|
||||
# can suggest other relevant pages even when one ranked highest.
|
||||
assert "docs index" in text
|
||||
assert "deployment.mdx" in text
|
||||
|
||||
def test_truncates_to_max_chars(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
text = DocsReference().build_text("Datadog", max_chars=120, root=tmp_path)
|
||||
assert len(text) <= 200
|
||||
assert "truncated" in text
|
||||
|
||||
|
||||
class TestBuildDocsIndex:
|
||||
def test_lists_all_pages_with_titles(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
pages = DocsReference().discover(tmp_path)
|
||||
index = build_docs_index(pages)
|
||||
assert "datadog.mdx: Datadog" in index
|
||||
assert "deployment.mdx: Deployment" in index
|
||||
|
||||
def test_returns_empty_string_for_no_pages(self) -> None:
|
||||
assert build_docs_index([]) == ""
|
||||
|
||||
|
||||
class TestExcerpt:
|
||||
def test_returns_full_body_when_short(self) -> None:
|
||||
body = "Short body."
|
||||
assert excerpt(body, max_chars=100) == "Short body."
|
||||
|
||||
def test_truncates_long_body_with_marker(self) -> None:
|
||||
body = ("paragraph one. " * 10) + "\n\n" + ("paragraph two. " * 10)
|
||||
out = excerpt(body, max_chars=80)
|
||||
assert "truncated" in out
|
||||
|
||||
|
||||
class TestDocPageDataclass:
|
||||
def test_is_hashable_and_immutable(self) -> None:
|
||||
page = DocPage(slug="x", relpath="x.mdx", title="X", body="hello")
|
||||
# frozen dataclasses are hashable, so they can be stored in sets.
|
||||
assert page in {page}
|
||||
|
||||
|
||||
class TestDocsGroundingCache:
|
||||
def test_cache_maxsize_matches_implementation(self) -> None:
|
||||
stats = DocsReference().stats()
|
||||
assert stats.maxsize == 32
|
||||
|
||||
def test_repeated_discover_hits_parse_cache(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
ref = DocsReference()
|
||||
ref.discover(tmp_path)
|
||||
info1 = ref.stats()
|
||||
ref.discover(tmp_path)
|
||||
info2 = ref.stats()
|
||||
assert info2.hits == info1.hits + 1
|
||||
assert info2.misses == info1.misses
|
||||
|
||||
def test_invalidate_resets_stats(self, tmp_path: Path) -> None:
|
||||
_seed_docs(tmp_path)
|
||||
ref = DocsReference()
|
||||
ref.discover(tmp_path)
|
||||
ref.discover(tmp_path)
|
||||
assert ref.stats().hits >= 1
|
||||
ref.invalidate()
|
||||
cleared = ref.stats()
|
||||
assert cleared.hits == 0
|
||||
assert cleared.misses == 0
|
||||
assert cleared.currsize == 0
|
||||
|
||||
def test_file_edit_invalidates_and_refreshes_content(self, tmp_path: Path) -> None:
|
||||
ref = DocsReference()
|
||||
_write_doc(
|
||||
tmp_path,
|
||||
"datadog.mdx",
|
||||
'---\ntitle: "Datadog"\n---\n\nOld content.\n',
|
||||
)
|
||||
pages1 = ref.discover(tmp_path)
|
||||
assert any("Old content" in p.body for p in pages1)
|
||||
|
||||
datadog = tmp_path / "datadog.mdx"
|
||||
datadog.write_text(
|
||||
'---\ntitle: "Datadog"\n---\n\nNew refreshed content.\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
pages2 = ref.discover(tmp_path)
|
||||
assert any("New refreshed content" in p.body for p in pages2)
|
||||
assert not any("Old content" in p.body for p in pages2)
|
||||
|
||||
def test_single_tree_walk_per_discover_call(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Regression: avoid a second full walk inside the parse path on a miss."""
|
||||
_seed_docs(tmp_path)
|
||||
calls = 0
|
||||
real_iter = docs_reference._iter_doc_files
|
||||
|
||||
def _spy(root: Path) -> list[Path]:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return real_iter(root)
|
||||
|
||||
monkeypatch.setattr(docs_reference, "_iter_doc_files", _spy)
|
||||
ref = DocsReference()
|
||||
ref.discover(tmp_path)
|
||||
assert calls == 1
|
||||
ref.discover(tmp_path)
|
||||
assert calls == 2
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Tests for optional grounding-cache diagnostic logging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
|
||||
from core.agent_harness.grounding.diagnostics import (
|
||||
GroundingSource,
|
||||
log_grounding_cache_diagnostics,
|
||||
)
|
||||
from core.agent_harness.grounding.models import CacheStats
|
||||
|
||||
|
||||
def _sources() -> list[GroundingSource]:
|
||||
return [GroundingSource(name="unit", stats_fn=lambda: CacheStats(name="unit", hits=1))]
|
||||
|
||||
|
||||
def test_log_skips_when_tracer_verbose_unset(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.delenv("TRACER_VERBOSE", raising=False)
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
log_grounding_cache_diagnostics(_sources(), "unit_test")
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_log_skips_when_tracer_verbose_not_one(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.setenv("TRACER_VERBOSE", "0")
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
log_grounding_cache_diagnostics(_sources(), "unit_test")
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_log_emits_debug_when_tracer_verbose_on(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
monkeypatch.setenv("TRACER_VERBOSE", "1")
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
log_grounding_cache_diagnostics(_sources(), "unit_test_reason")
|
||||
assert any("unit_test_reason" in r.message for r in caplog.records)
|
||||
assert any("grounding cache" in r.message.lower() for r in caplog.records)
|
||||
Reference in New Issue
Block a user