wehub-resource-sync
133 lines
5.6 KiB
Python
133 lines
5.6 KiB
Python
"""Tests for the harness_portability layer."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from plugin_eval.layers.harness_portability import (
|
|
detect_agent_findings,
|
|
detect_skill_findings,
|
|
score_agent_portability,
|
|
score_skill_portability,
|
|
)
|
|
from plugin_eval.parser import parse_agent, parse_skill
|
|
|
|
|
|
def _make_skill(tmp_path: Path, body: str, *, name: str = "test-skill", references: bool = False) -> Path:
|
|
skill_dir = tmp_path / name
|
|
skill_dir.mkdir()
|
|
(skill_dir / "SKILL.md").write_text(
|
|
f"---\nname: {name}\ndescription: Use when testing portability.\n---\n\n{body}\n"
|
|
)
|
|
if references:
|
|
(skill_dir / "references").mkdir()
|
|
(skill_dir / "references" / "details.md").write_text("Detail.\n")
|
|
return skill_dir
|
|
|
|
|
|
def _make_agent(tmp_path: Path, frontmatter: str, body: str = "Body.\n") -> Path:
|
|
agent_file = tmp_path / "agent.md"
|
|
agent_file.write_text(f"---\n{frontmatter}\n---\n\n{body}")
|
|
return agent_file
|
|
|
|
|
|
class TestSkillFindings:
|
|
def test_clean_skill_has_no_findings(self, tmp_path: Path):
|
|
skill_dir = _make_skill(tmp_path, "Use action verbs; run a shell command.")
|
|
skill = parse_skill(skill_dir)
|
|
assert detect_skill_findings(skill) == []
|
|
assert score_skill_portability(skill) == 1.0
|
|
|
|
def test_skill_over_codex_cap_fires_when_no_references(self, tmp_path: Path):
|
|
skill_dir = _make_skill(tmp_path, "x" * 9000, references=False)
|
|
skill = parse_skill(skill_dir)
|
|
flags = [f.flag for f in detect_skill_findings(skill)]
|
|
assert "SKILL_OVER_CODEX_CAP" in flags
|
|
|
|
def test_skill_over_codex_cap_suppressed_when_references_exist(self, tmp_path: Path):
|
|
skill_dir = _make_skill(tmp_path, "x" * 9000, references=True)
|
|
skill = parse_skill(skill_dir)
|
|
flags = [f.flag for f in detect_skill_findings(skill)]
|
|
assert "SKILL_OVER_CODEX_CAP" not in flags
|
|
|
|
def test_camel_tool_refs_fire(self, tmp_path: Path):
|
|
# Match a Claude-tool reference in context: "use `Read`", "the `Bash`", or "`Edit` tool"
|
|
skill_dir = _make_skill(
|
|
tmp_path, "Use `Read` to open files. Call `Edit` to modify. The `Bash` tool runs commands."
|
|
)
|
|
skill = parse_skill(skill_dir)
|
|
flags = [f.flag for f in detect_skill_findings(skill)]
|
|
assert "CLAUDE_TOOL_REFS" in flags
|
|
|
|
def test_bare_backticked_token_no_false_positive(self, tmp_path: Path):
|
|
"""Generic prose backticks like Rust's `Task` type must NOT fire CLAUDE_TOOL_REFS."""
|
|
skill_dir = _make_skill(
|
|
tmp_path,
|
|
"Rust's `Task` is a future. `Read` and `Write` are stdlib traits. Set `LS` for line sep.",
|
|
)
|
|
skill = parse_skill(skill_dir)
|
|
flags = [f.flag for f in detect_skill_findings(skill)]
|
|
assert "CLAUDE_TOOL_REFS" not in flags
|
|
|
|
def test_tool_prose_fires(self, tmp_path: Path):
|
|
skill_dir = _make_skill(tmp_path, "First, use the Read tool to open the file.")
|
|
skill = parse_skill(skill_dir)
|
|
flags = [f.flag for f in detect_skill_findings(skill)]
|
|
assert "CLAUDE_TOOL_PROSE" in flags
|
|
|
|
def test_tool_prose_no_false_positive_on_lowercase_words(self, tmp_path: Path):
|
|
"""`the bash tool` (shell, lowercase) must NOT fire CLAUDE_TOOL_PROSE."""
|
|
skill_dir = _make_skill(
|
|
tmp_path,
|
|
"Configure the bash tool in your Makefile. The read tool target is at line 12.",
|
|
)
|
|
skill = parse_skill(skill_dir)
|
|
flags = [f.flag for f in detect_skill_findings(skill)]
|
|
assert "CLAUDE_TOOL_PROSE" not in flags
|
|
|
|
def test_findings_carry_remediation(self, tmp_path: Path):
|
|
skill_dir = _make_skill(tmp_path, "Use the Bash tool.")
|
|
skill = parse_skill(skill_dir)
|
|
findings = detect_skill_findings(skill)
|
|
assert findings
|
|
assert all(f.remediation for f in findings)
|
|
# Remediation appears in the AntiPattern description
|
|
ap = findings[0].to_anti_pattern()
|
|
assert "Fix:" in ap.description
|
|
|
|
|
|
class TestAgentFindings:
|
|
def test_clean_agent_has_no_findings(self, tmp_path: Path):
|
|
agent_file = _make_agent(tmp_path, "name: my-explorer\ndescription: Use when exploring.\nmodel: inherit")
|
|
agent = parse_agent(agent_file)
|
|
assert detect_agent_findings(agent) == []
|
|
assert score_agent_portability(agent) == 1.0
|
|
|
|
def test_builtin_name_collision_fires(self, tmp_path: Path):
|
|
agent_file = _make_agent(tmp_path, "name: worker\ndescription: Use when working.\nmodel: inherit")
|
|
agent = parse_agent(agent_file)
|
|
flags = [f.flag for f in detect_agent_findings(agent)]
|
|
assert "AGENT_NAME_COLLISION" in flags
|
|
|
|
def test_bare_model_alias_fires(self, tmp_path: Path):
|
|
agent_file = _make_agent(tmp_path, "name: my-agent\ndescription: Use when delegating.\nmodel: opus")
|
|
agent = parse_agent(agent_file)
|
|
flags = [f.flag for f in detect_agent_findings(agent)]
|
|
assert "BARE_MODEL_ALIAS" in flags
|
|
|
|
def test_inherit_model_passes(self, tmp_path: Path):
|
|
agent_file = _make_agent(tmp_path, "name: my-agent\ndescription: Use when delegating.\nmodel: inherit")
|
|
agent = parse_agent(agent_file)
|
|
flags = [f.flag for f in detect_agent_findings(agent)]
|
|
assert "BARE_MODEL_ALIAS" not in flags
|
|
|
|
def test_camel_tool_refs_in_agent_body(self, tmp_path: Path):
|
|
agent_file = _make_agent(
|
|
tmp_path,
|
|
"name: my-agent\ndescription: Use when delegating.\nmodel: inherit",
|
|
body="Use `Read` and `Glob` to explore the filesystem.",
|
|
)
|
|
agent = parse_agent(agent_file)
|
|
flags = [f.flag for f in detect_agent_findings(agent)]
|
|
assert "CLAUDE_TOOL_REFS" in flags
|