wehub-resource-sync
208 lines
8.1 KiB
Python
208 lines
8.1 KiB
Python
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from plugin_eval.layers.static import _TRIGGER_PATTERN, StaticAnalyzer
|
|
from plugin_eval.models import LayerResult
|
|
|
|
|
|
def _make_skill(tmp_path: Path, description: str, name: str = "test-skill") -> Path:
|
|
skill_dir = tmp_path / name
|
|
skill_dir.mkdir()
|
|
(skill_dir / "SKILL.md").write_text(
|
|
f"---\nname: {name}\ndescription: \"{description}\"\n---\n\n"
|
|
"# Skill\n\n## Overview\n\nBody.\n"
|
|
)
|
|
return skill_dir
|
|
|
|
|
|
class TestStaticAnalyzer:
|
|
def test_analyze_valid_skill(self, sample_skill_dir: Path):
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_skill(sample_skill_dir)
|
|
assert isinstance(result, LayerResult)
|
|
assert result.layer == "static"
|
|
assert result.score > 0.5
|
|
assert len(result.anti_patterns) == 0
|
|
|
|
def test_analyze_poor_skill(self, poor_skill_dir: Path):
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_skill(poor_skill_dir)
|
|
assert result.score < 0.7
|
|
flags = [ap.flag for ap in result.anti_patterns]
|
|
assert "OVER_CONSTRAINED" in flags
|
|
assert "MISSING_TRIGGER" in flags
|
|
|
|
def test_analyze_plugin(self, sample_plugin_dir: Path):
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_plugin(sample_plugin_dir)
|
|
assert result.layer == "static"
|
|
assert result.score > 0.5
|
|
assert "skill_scores" in result.sub_scores
|
|
assert "agent_scores" in result.sub_scores
|
|
|
|
def test_anti_pattern_penalty(self):
|
|
analyzer = StaticAnalyzer()
|
|
assert analyzer._anti_pattern_penalty(0) == 1.0
|
|
assert analyzer._anti_pattern_penalty(2) == pytest.approx(0.9)
|
|
assert analyzer._anti_pattern_penalty(10) == 0.5
|
|
assert analyzer._anti_pattern_penalty(20) == 0.5
|
|
|
|
def test_description_pushiness_score(self):
|
|
analyzer = StaticAnalyzer()
|
|
good = "Test skill for evaluation. Use when testing plugin-eval. Use PROACTIVELY for quality checks."
|
|
weak = "A skill."
|
|
assert analyzer._description_pushiness(good) > analyzer._description_pushiness(weak)
|
|
|
|
|
|
class TestTriggerPattern:
|
|
"""Regression coverage for the broadened trigger-phrase matcher.
|
|
|
|
plugin-dev's canonical recommendation is third-person ("This skill should be
|
|
used when …"), and several real-world plugins use prepositional triggers
|
|
("Use after …", "Use before …"). The pre-2026 regex only matched the
|
|
imperative "Use when …" form, which produced false-positive MISSING_TRIGGER
|
|
flags against Anthropic's own examples.
|
|
"""
|
|
|
|
@pytest.mark.parametrize(
|
|
"description",
|
|
[
|
|
"Use when testing plugin-eval.",
|
|
"Use this skill when scaffolding plugins.",
|
|
"This skill should be used when the user asks to 'create a hook'.",
|
|
"Used when several attempts have failed in a row.",
|
|
"Use after editing the source-of-truth files, before committing.",
|
|
"Use before declaring a task complete after a hard debugging session.",
|
|
"Use immediately before a commit, push, or edit-after-failure.",
|
|
"Use whenever you are asked to plan inside a Paperclip company.",
|
|
"Auto-loads when working on test files.",
|
|
"Trigger when a Bash command fails three times in a row.",
|
|
"Use PROACTIVELY before merging.",
|
|
],
|
|
)
|
|
def test_pattern_matches_canonical_forms(self, description: str) -> None:
|
|
assert _TRIGGER_PATTERN.search(description), (
|
|
f"Expected trigger phrase to match in: {description!r}"
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
"description",
|
|
[
|
|
"A skill.",
|
|
"Provides hook guidance.",
|
|
"Returns the current timestamp.",
|
|
"Performs static analysis on plugin directories.",
|
|
],
|
|
)
|
|
def test_pattern_rejects_descriptions_without_trigger(self, description: str) -> None:
|
|
assert not _TRIGGER_PATTERN.search(description), (
|
|
f"Did not expect trigger match in: {description!r}"
|
|
)
|
|
|
|
def test_third_person_skill_does_not_flag_missing_trigger(self, tmp_path: Path) -> None:
|
|
"""Anthropic plugin-dev's canonical phrasing must not be flagged."""
|
|
skill_dir = _make_skill(
|
|
tmp_path,
|
|
"This skill should be used when the user asks to 'create a hook', "
|
|
"'add a PreToolUse hook', or 'validate tool use'.",
|
|
)
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_skill(skill_dir)
|
|
flags = [ap.flag for ap in result.anti_patterns]
|
|
assert "MISSING_TRIGGER" not in flags
|
|
|
|
def test_prepositional_trigger_does_not_flag_missing_trigger(self, tmp_path: Path) -> None:
|
|
skill_dir = _make_skill(
|
|
tmp_path,
|
|
"Self-check before a single risky action. Use immediately before a "
|
|
"commit, push, edit-after-failure, or skip-a-verification step.",
|
|
)
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_skill(skill_dir)
|
|
flags = [ap.flag for ap in result.anti_patterns]
|
|
assert "MISSING_TRIGGER" not in flags
|
|
|
|
|
|
def _make_skill_with_frontmatter(
|
|
tmp_path: Path, frontmatter_lines: list[str], name: str = "test-skill"
|
|
) -> Path:
|
|
skill_dir = tmp_path / name
|
|
skill_dir.mkdir()
|
|
frontmatter = "\n".join(frontmatter_lines)
|
|
(skill_dir / "SKILL.md").write_text(
|
|
f"---\n{frontmatter}\n---\n\n# Skill\n\n## Overview\n\nBody.\n"
|
|
)
|
|
return skill_dir
|
|
|
|
|
|
class TestTriggerExemptions:
|
|
"""`disable-model-invocation: true` and `paths:` frontmatter should exempt
|
|
a skill from the MISSING_TRIGGER check, because those skills are not
|
|
auto-invoked from the description.
|
|
"""
|
|
|
|
def test_disable_model_invocation_exempts_skill(self, tmp_path: Path) -> None:
|
|
skill_dir = _make_skill_with_frontmatter(
|
|
tmp_path,
|
|
[
|
|
"name: setup",
|
|
"description: One-time setup that adds .claude/state/ to the project's .gitignore.",
|
|
"disable-model-invocation: true",
|
|
],
|
|
)
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_skill(skill_dir)
|
|
flags = [ap.flag for ap in result.anti_patterns]
|
|
assert "MISSING_TRIGGER" not in flags, (
|
|
"Slash-only skills should not be flagged for missing description trigger"
|
|
)
|
|
|
|
def test_paths_auto_load_exempts_skill(self, tmp_path: Path) -> None:
|
|
skill_dir = _make_skill_with_frontmatter(
|
|
tmp_path,
|
|
[
|
|
"name: self-evaluate",
|
|
"description: Self-critical evaluation guard for test/spec files.",
|
|
'paths: "**/*test*,**/*spec*"',
|
|
],
|
|
)
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_skill(skill_dir)
|
|
flags = [ap.flag for ap in result.anti_patterns]
|
|
assert "MISSING_TRIGGER" not in flags, (
|
|
"Path-triggered skills should not be flagged for missing description trigger"
|
|
)
|
|
|
|
def test_disable_model_invocation_false_still_checks_trigger(
|
|
self, tmp_path: Path
|
|
) -> None:
|
|
skill_dir = _make_skill_with_frontmatter(
|
|
tmp_path,
|
|
[
|
|
"name: model-invocable",
|
|
"description: A description without a trigger phrase whatsoever.",
|
|
"disable-model-invocation: false",
|
|
],
|
|
)
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_skill(skill_dir)
|
|
flags = [ap.flag for ap in result.anti_patterns]
|
|
assert "MISSING_TRIGGER" in flags, (
|
|
"Model-invocable skills without a trigger phrase must still be flagged"
|
|
)
|
|
|
|
def test_empty_paths_value_still_checks_trigger(self, tmp_path: Path) -> None:
|
|
skill_dir = _make_skill_with_frontmatter(
|
|
tmp_path,
|
|
[
|
|
"name: bad-paths",
|
|
"description: Some skill description that lacks the trigger phrase.",
|
|
'paths: ""',
|
|
],
|
|
)
|
|
analyzer = StaticAnalyzer()
|
|
result = analyzer.analyze_skill(skill_dir)
|
|
flags = [ap.flag for ap in result.anti_patterns]
|
|
assert "MISSING_TRIGGER" in flags
|