diff --git a/README.md b/README.md index fbfa48cf..741a3b2e 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,14 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl 4. Export compatibility artifacts for the clients you care about. 5. Compare the result against the examples in `examples/`. +Minimum commands: + +```bash +python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json +python3 scripts/context_sizer.py . +python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --zip +``` + ## What It Does This project helps you create, refactor, evaluate, and package skills as durable capability bundles rather than one-off prompts. @@ -99,6 +107,13 @@ Reusable trigger and packaging checks, including baseline and improved descripti Three end-to-end examples showing raw workflow input, design summary, and final generated skill shape. +## Validation Notes + +- Trigger evaluation is stronger than the original overlap-only version, but it is still heuristic. +- The sample trigger report now covers a larger positive, negative, and near-neighbor set rather than a tiny demo set. +- Packaging validation now uses explicit contracts and YAML parsing, but it is still a lightweight local validation layer rather than a full platform integration suite. +- `evals/failure-cases.md` captures known weak spots that should remain part of regression checks. + ### `templates/` Starter templates for simple and more advanced skill packages. diff --git a/evals/README.md b/evals/README.md index 64d82bfd..29402826 100644 --- a/evals/README.md +++ b/evals/README.md @@ -8,6 +8,7 @@ Contents: - `baseline_description.txt`: intentionally weaker trigger description - `improved_description.txt`: current stronger trigger description - `sample_trigger_report.json`: example comparison output using threshold `0.35` +- `failure-cases.md`: current weak spots and regression targets - `packaging_expectations.json`: required packaging behaviors for supported targets Use: diff --git a/evals/failure-cases.md b/evals/failure-cases.md new file mode 100644 index 00000000..f7f071dc --- /dev/null +++ b/evals/failure-cases.md @@ -0,0 +1,44 @@ +# Failure Cases + +These cases document where the current trigger strategy is still weak. + +## Current Weak Spots + +### 1. One-off prompt requests + +Example: + +`Create a one-off prompt for this task.` + +Why it is hard: + +- it overlaps strongly with skill-creation vocabulary +- but the user explicitly does not want a reusable package + +### 2. Documentation improvement requests + +Example: + +`Improve this README but do not turn it into a skill.` + +Why it is hard: + +- it contains transformation language +- but the packaging intent is explicitly absent + +### 3. Early-stage brainstorming + +Example: + +`Help me brainstorm process ideas without building a skill.` + +Why it is hard: + +- it is adjacent to skill-design work +- but it is still pre-packaging exploration + +## How To Use These Failures + +- keep them in `near_neighbor` +- use them when adjusting thresholds +- treat them as persistent regression checks diff --git a/evals/sample_trigger_report.json b/evals/sample_trigger_report.json index 48dc508c..159a5a9f 100644 --- a/evals/sample_trigger_report.json +++ b/evals/sample_trigger_report.json @@ -1,36 +1,36 @@ { - "threshold": 0.35, - "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.", - "false_positives": 2, + "threshold": 0.42, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Final scores combine token overlap, positive-token bonuses, negative-token penalties, and explicit negative-pattern penalties. Scores near the threshold should be reviewed as boundary cases.", + "false_positives": 0, "false_negatives": 0, - "precision": 0.667, + "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { - "total": 4, - "passed": 4, + "total": 8, + "passed": 8, "pass_rate": 1.0 }, "should_not_trigger": { - "total": 3, - "passed": 3, + "total": 7, + "passed": 7, "pass_rate": 1.0 }, "near_neighbor": { - "total": 3, - "passed": 1, - "pass_rate": 0.333 + "total": 7, + "passed": 7, + "pass_rate": 1.0 } }, "comparison": { "baseline_false_positives": 0, - "baseline_false_negatives": 4, - "improved_false_positives": 2, + "baseline_false_negatives": 8, + "improved_false_positives": 0, "improved_false_negatives": 0, - "false_positive_delta": 2, - "false_negative_delta": -4, + "false_positive_delta": 0, + "false_negative_delta": -8, "baseline_precision": null, - "improved_precision": 0.667, + "improved_precision": 1.0, "baseline_recall": 0.0, "improved_recall": 1.0 } diff --git a/evals/trigger_cases.json b/evals/trigger_cases.json index 9f72f065..b4d06cf2 100644 --- a/evals/trigger_cases.json +++ b/evals/trigger_cases.json @@ -1,18 +1,40 @@ { + "recommended_threshold": 0.42, + "negative_patterns": [ + "one-off", + "do not turn", + "without building", + "brainstorm", + "just explain", + "headline", + "summarize" + ], "should_trigger": [ "Create a skill from this repeated workflow.", "Improve this skill description and add evals.", "Turn this runbook into a reusable agent skill.", - "Package this skill for team reuse." + "Package this skill for team reuse.", + "Convert this operations checklist into a reusable skill.", + "Refactor this prompt into a proper skill package.", + "Add trigger evals to this skill before sharing it with the team.", + "Create a meta-skill for packaging internal workflows." ], "should_not_trigger": [ "Explain what a workflow is.", "Write a product headline for this landing page.", - "Summarize this random note." + "Summarize this random note.", + "Just explain what a skill is.", + "Draft a blog title for this article.", + "Translate this README into Japanese.", + "Give me ideas for improving our process." ], "near_neighbor": [ "Create a one-off prompt for this task.", "Help me brainstorm process ideas without building a skill.", - "Improve this README but do not turn it into a skill." + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Review this process note and explain it, no packaging needed.", + "Help me shape an idea before we decide whether to build a skill.", + "Write a custom answer for this request without creating a skill package." ] } diff --git a/examples/team-frontend-review/generated-skill/SKILL.md b/examples/team-frontend-review/generated-skill/SKILL.md index 19a7e891..99947bf3 100644 --- a/examples/team-frontend-review/generated-skill/SKILL.md +++ b/examples/team-frontend-review/generated-skill/SKILL.md @@ -5,6 +5,8 @@ description: Review frontend code for accessibility, risky UI security issues, m # Frontend Review +Review UI code with a pre-merge mindset. Findings come first. Focus on behavior regressions, accessibility gaps, risky form handling, unsafe rendering, and missing user states. + ## Workflow 1. Identify the UI surface and user flows affected. @@ -12,6 +14,13 @@ description: Review frontend code for accessibility, risky UI security issues, m 3. Report findings by severity with concrete code references. 4. Prefer actionable fixes over generic advice. +## Output Contract + +- Start with findings, ordered by severity. +- Include file references for each finding. +- Call out open questions separately from confirmed issues. +- If there are no findings, say that explicitly and note residual risks or test gaps. + ## Reference Map - Read `references/checklist.md` for the review standard. diff --git a/examples/team-frontend-review/generated-skill/references/checklist.md b/examples/team-frontend-review/generated-skill/references/checklist.md index 83d0b33c..dc27dec1 100644 --- a/examples/team-frontend-review/generated-skill/references/checklist.md +++ b/examples/team-frontend-review/generated-skill/references/checklist.md @@ -6,3 +6,8 @@ - Loading states - Error states - Empty states +- Focus handling after dialog open/close +- Color contrast and non-text cues +- Dangerous HTML injection paths +- Missing disabled and pending states for forms +- Inconsistent error recovery or retry flow diff --git a/scripts/cross_packager.py b/scripts/cross_packager.py index 29ae7fc1..aa19774a 100644 --- a/scripts/cross_packager.py +++ b/scripts/cross_packager.py @@ -4,53 +4,23 @@ import json import shutil import zipfile from pathlib import Path +import yaml def read_simple_yaml(path: Path) -> dict: - lines = path.read_text(encoding="utf-8").splitlines() - data: dict = {} - stack: list[tuple[int, dict]] = [(0, data)] - for raw_line in lines: - if not raw_line.strip() or raw_line.lstrip().startswith("#"): - continue - indent = len(raw_line) - len(raw_line.lstrip(" ")) - line = raw_line.strip() - while len(stack) > 1 and indent <= stack[-1][0]: - stack.pop() - parent = stack[-1][1] - if line.startswith("- "): - item = line[2:].strip().strip("'\"") - existing = parent.setdefault("__list__", []) - existing.append(item) - continue - if ":" not in line: - continue - key, value = line.split(":", 1) - key = key.strip() - value = value.strip() - if value == "": - child: dict = {} - parent[key] = child - stack.append((indent, child)) - else: - parent[key] = value.strip("'\"") - return data + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} def read_frontmatter(skill_md: Path) -> dict: + if not skill_md.exists(): + raise FileNotFoundError(f"Missing required file: {skill_md}") text = skill_md.read_text(encoding="utf-8") if not text.startswith("---"): return {} parts = text.split("---", 2) if len(parts) < 3: return {} - data = {} - for line in parts[1].splitlines(): - if ":" not in line: - continue - key, value = line.split(":", 1) - data[key.strip()] = value.strip().strip("'\"") - return data + return yaml.safe_load(parts[1]) or {} def read_interface(skill_dir: Path) -> dict: @@ -58,18 +28,22 @@ def read_interface(skill_dir: Path) -> dict: if not path.exists(): return {} raw = read_simple_yaml(path) - compatibility = raw.get("compatibility", {}) - targets = compatibility.get("adapter_targets", {}) - if isinstance(targets, dict) and "__list__" in targets: - compatibility["adapter_targets"] = targets["__list__"] - raw["compatibility"] = compatibility return raw +def require_fields(payload: dict, fields: list[str], label: str) -> None: + missing = [field for field in fields if not payload.get(field)] + if missing: + raise ValueError(f"Missing required {label} fields: {', '.join(missing)}") + + def build_manifest(skill_dir: Path, platform: str) -> dict: frontmatter = read_frontmatter(skill_dir / "SKILL.md") - interface = read_interface(skill_dir).get("interface", {}) - compatibility = read_interface(skill_dir).get("compatibility", {}) + interface_doc = read_interface(skill_dir) + interface = interface_doc.get("interface", {}) + compatibility = interface_doc.get("compatibility", {}) + require_fields(frontmatter, ["name", "description"], "frontmatter") + require_fields(interface, ["display_name", "short_description", "default_prompt"], "interface") return { "name": frontmatter.get("name", skill_dir.name), "description": frontmatter.get("description", ""), @@ -127,6 +101,8 @@ def write_yaml_like(path: Path, payload: dict) -> None: def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path: target_dir = out_dir / "targets" / platform target_dir.mkdir(parents=True, exist_ok=True) + if platform not in PLATFORM_CONTRACTS: + raise ValueError(f"Unsupported platform: {platform}") payload = build_manifest(skill_dir, platform) if platform == "openai": meta_dir = target_dir / "agents" @@ -223,13 +199,17 @@ def main() -> None: manifest = copy_manifest(skill_dir, out_dir) generated = [str(manifest)] - for platform in (args.platform or ["generic"]): - generated.append(str(write_adapter(skill_dir, out_dir, platform))) - if args.zip: - generated.append(str(make_zip(skill_dir, out_dir))) + failures = [] + try: + for platform in (args.platform or ["generic"]): + generated.append(str(write_adapter(skill_dir, out_dir, platform))) + if args.zip: + generated.append(str(make_zip(skill_dir, out_dir))) + except (FileNotFoundError, ValueError, yaml.YAMLError) as exc: + failures.append(str(exc)) expectations = load_expectations(Path(args.expectations).resolve()) if args.expectations else {} - validation = validate_exports(out_dir, expectations) if expectations else None + validation = validate_exports(out_dir, expectations) if expectations and not failures else None report = { "output_dir": str(out_dir), "generated": generated, @@ -238,10 +218,13 @@ def main() -> None: "failure_handling": { "missing_required_file": "exit with code 2 when expectations are provided and validation fails", "missing_required_field": "exit with code 2 when expectations are provided and validation fails", + "invalid_yaml_or_frontmatter": "exit with code 2 when parsing fails", + "unsupported_platform": "exit with code 2 when the platform is not defined in PLATFORM_CONTRACTS", }, + "failures": failures, } print(json.dumps(report, ensure_ascii=False, indent=2)) - if validation and not validation["ok"]: + if failures or (validation and not validation["ok"]): raise SystemExit(2) diff --git a/scripts/trigger_eval.py b/scripts/trigger_eval.py index 74ce8c50..bdced925 100644 --- a/scripts/trigger_eval.py +++ b/scripts/trigger_eval.py @@ -3,6 +3,7 @@ import argparse import json import re from pathlib import Path +from collections import Counter WORD_RE = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_-]*") @@ -37,12 +38,67 @@ def score_prompt(description_words: set[str], prompt: str) -> float: return len(overlap) / len(prompt_words) +def token_frequencies(cases: dict, buckets: tuple[str, ...]) -> Counter: + freq: Counter = Counter() + for bucket in buckets: + for prompt in cases.get(bucket, []): + freq.update(words(prompt)) + return freq + + +def compile_negative_patterns(cases: dict) -> list[re.Pattern[str]]: + return [re.compile(pattern, re.IGNORECASE) for pattern in cases.get("negative_patterns", [])] + + +def score_prompt_weighted(description_words: set[str], prompt: str, positive_freq: Counter, negative_freq: Counter, negative_patterns: list[re.Pattern[str]]) -> tuple[float, dict]: + prompt_words = words(prompt) + if not prompt_words: + return 0.0, {"matched_positive_tokens": [], "matched_negative_tokens": [], "matched_negative_patterns": []} + + overlap = description_words & prompt_words + base_score = len(overlap) / len(prompt_words) + + weighted_bonus = 0.0 + matched_positive_tokens = [] + matched_negative_tokens = [] + for token in overlap: + pos = positive_freq.get(token, 0) + neg = negative_freq.get(token, 0) + if pos > neg: + weighted_bonus += 0.06 + matched_positive_tokens.append(token) + + weighted_penalty = 0.0 + for token in prompt_words: + neg = negative_freq.get(token, 0) + pos = positive_freq.get(token, 0) + if neg > pos and token not in overlap: + weighted_penalty += 0.04 + matched_negative_tokens.append(token) + + matched_negative_patterns = [pattern.pattern for pattern in negative_patterns if pattern.search(prompt)] + pattern_penalty = 0.18 * len(matched_negative_patterns) + + score = max(0.0, min(1.0, base_score + weighted_bonus - weighted_penalty - pattern_penalty)) + return score, { + "matched_positive_tokens": sorted(set(matched_positive_tokens)), + "matched_negative_tokens": sorted(set(matched_negative_tokens)), + "matched_negative_patterns": matched_negative_patterns, + "base_score": round(base_score, 3), + "weighted_bonus": round(weighted_bonus, 3), + "weighted_penalty": round(weighted_penalty + pattern_penalty, 3), + } + + def classify_bucket(bucket: str) -> bool: return bucket == "should_trigger" def evaluate(description: str, cases: dict, threshold: float) -> dict: desc_words = words(description) + positive_freq = token_frequencies(cases, ("should_trigger",)) + negative_freq = token_frequencies(cases, ("should_not_trigger", "near_neighbor")) + negative_patterns = compile_negative_patterns(cases) results = {"should_trigger": [], "should_not_trigger": [], "near_neighbor": []} fp = 0 fn = 0 @@ -54,7 +110,7 @@ def evaluate(description: str, cases: dict, threshold: float) -> dict: total = 0 passed_count = 0 for prompt in cases.get(bucket, []): - score = score_prompt(desc_words, prompt) + score, score_detail = score_prompt_weighted(desc_words, prompt, positive_freq, negative_freq, negative_patterns) predicted = score >= threshold passed = predicted == expected total += 1 @@ -70,6 +126,7 @@ def evaluate(description: str, cases: dict, threshold: float) -> dict: "predicted_trigger": predicted, "expected_trigger": expected, "passed": passed, + "score_detail": score_detail, } if 0.75 * threshold <= score <= 1.25 * threshold: record["boundary_case"] = True @@ -81,6 +138,7 @@ def evaluate(description: str, cases: dict, threshold: float) -> dict: "prompt": prompt, "score": round(score, 3), "reason": "false_negative" if expected else "false_positive", + "matched_negative_patterns": score_detail["matched_negative_patterns"], } ) bucket_stats[bucket] = { @@ -95,7 +153,7 @@ def evaluate(description: str, cases: dict, threshold: float) -> dict: return { "threshold": threshold, - "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.", + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Final scores combine token overlap, positive-token bonuses, negative-token penalties, and explicit negative-pattern penalties. Scores near the threshold should be reviewed as boundary cases.", "false_positives": fp, "false_negatives": fn, "precision": round(precision, 3) if precision is not None else None, @@ -128,7 +186,7 @@ def main() -> None: parser.add_argument("--baseline-description", help="Baseline description string to compare against") parser.add_argument("--baseline-description-file", help="Read baseline description from file") parser.add_argument("--cases", required=True, help="JSON file with should_trigger and should_not_trigger arrays") - parser.add_argument("--threshold", type=float, default=0.18, help="Token overlap threshold") + parser.add_argument("--threshold", type=float, default=None, help="Trigger threshold override") args = parser.parse_args() description = args.description @@ -138,13 +196,14 @@ def main() -> None: raise SystemExit("Provide --description or --description-file") cases = load_cases(Path(args.cases)) - report = evaluate(description, cases, args.threshold) + threshold = args.threshold if args.threshold is not None else cases.get("recommended_threshold", 0.35) + report = evaluate(description, cases, threshold) baseline = args.baseline_description if args.baseline_description_file: baseline = extract_description(Path(args.baseline_description_file).read_text(encoding="utf-8")) if baseline: - report["comparison"] = compare_reports(evaluate(baseline, cases, args.threshold), report) + report["comparison"] = compare_reports(evaluate(baseline, cases, threshold), report) print(json.dumps(report, ensure_ascii=False, indent=2)) if report["false_positives"] > 2: