feat: strengthen eval coverage and parsing reliability

README.md

@@ -29,6 +29,14 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
 4. Export compatibility artifacts for the clients you care about.
 5. Compare the result against the examples in `examples/`.
 
+Minimum commands:
+
+```bash
+python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json
+python3 scripts/context_sizer.py .
+python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --zip
+```
+
 ## What It Does
 
 This project helps you create, refactor, evaluate, and package skills as durable capability bundles rather than one-off prompts.
@@ -99,6 +107,13 @@ Reusable trigger and packaging checks, including baseline and improved descripti
 
 Three end-to-end examples showing raw workflow input, design summary, and final generated skill shape.
 
+## Validation Notes
+
+- Trigger evaluation is stronger than the original overlap-only version, but it is still heuristic.
+- The sample trigger report now covers a larger positive, negative, and near-neighbor set rather than a tiny demo set.
+- Packaging validation now uses explicit contracts and YAML parsing, but it is still a lightweight local validation layer rather than a full platform integration suite.
+- `evals/failure-cases.md` captures known weak spots that should remain part of regression checks.
+
 ### `templates/`
 
 Starter templates for simple and more advanced skill packages.

evals/README.md

@@ -8,6 +8,7 @@ Contents:
 - `baseline_description.txt`: intentionally weaker trigger description
 - `improved_description.txt`: current stronger trigger description
 - `sample_trigger_report.json`: example comparison output using threshold `0.35`
+- `failure-cases.md`: current weak spots and regression targets
 - `packaging_expectations.json`: required packaging behaviors for supported targets
 
 Use:

evals/failure-cases.md

@@ -0,0 +1,44 @@
+# Failure Cases
+
+These cases document where the current trigger strategy is still weak.
+
+## Current Weak Spots
+
+### 1. One-off prompt requests
+
+Example:
+
+`Create a one-off prompt for this task.`
+
+Why it is hard:
+
+- it overlaps strongly with skill-creation vocabulary
+- but the user explicitly does not want a reusable package
+
+### 2. Documentation improvement requests
+
+Example:
+
+`Improve this README but do not turn it into a skill.`
+
+Why it is hard:
+
+- it contains transformation language
+- but the packaging intent is explicitly absent
+
+### 3. Early-stage brainstorming
+
+Example:
+
+`Help me brainstorm process ideas without building a skill.`
+
+Why it is hard:
+
+- it is adjacent to skill-design work
+- but it is still pre-packaging exploration
+
+## How To Use These Failures
+
+- keep them in `near_neighbor`
+- use them when adjusting thresholds
+- treat them as persistent regression checks

evals/sample_trigger_report.json

@@ -1,36 +1,36 @@
 {
-  "threshold": 0.35,
-  "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.",
-  "false_positives": 2,
+  "threshold": 0.42,
+  "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Final scores combine token overlap, positive-token bonuses, negative-token penalties, and explicit negative-pattern penalties. Scores near the threshold should be reviewed as boundary cases.",
+  "false_positives": 0,
   "false_negatives": 0,
-  "precision": 0.667,
+  "precision": 1.0,
   "recall": 1.0,
   "bucket_stats": {
     "should_trigger": {
-      "total": 4,
-      "passed": 4,
+      "total": 8,
+      "passed": 8,
       "pass_rate": 1.0
     },
     "should_not_trigger": {
-      "total": 3,
-      "passed": 3,
+      "total": 7,
+      "passed": 7,
       "pass_rate": 1.0
     },
     "near_neighbor": {
-      "total": 3,
-      "passed": 1,
-      "pass_rate": 0.333
+      "total": 7,
+      "passed": 7,
+      "pass_rate": 1.0
     }
   },
   "comparison": {
     "baseline_false_positives": 0,
-    "baseline_false_negatives": 4,
-    "improved_false_positives": 2,
+    "baseline_false_negatives": 8,
+    "improved_false_positives": 0,
     "improved_false_negatives": 0,
-    "false_positive_delta": 2,
-    "false_negative_delta": -4,
+    "false_positive_delta": 0,
+    "false_negative_delta": -8,
     "baseline_precision": null,
-    "improved_precision": 0.667,
+    "improved_precision": 1.0,
     "baseline_recall": 0.0,
     "improved_recall": 1.0
   }

evals/trigger_cases.json

@@ -1,18 +1,40 @@
 {
+  "recommended_threshold": 0.42,
+  "negative_patterns": [
+    "one-off",
+    "do not turn",
+    "without building",
+    "brainstorm",
+    "just explain",
+    "headline",
+    "summarize"
+  ],
   "should_trigger": [
     "Create a skill from this repeated workflow.",
     "Improve this skill description and add evals.",
     "Turn this runbook into a reusable agent skill.",
-    "Package this skill for team reuse."
+    "Package this skill for team reuse.",
+    "Convert this operations checklist into a reusable skill.",
+    "Refactor this prompt into a proper skill package.",
+    "Add trigger evals to this skill before sharing it with the team.",
+    "Create a meta-skill for packaging internal workflows."
   ],
   "should_not_trigger": [
     "Explain what a workflow is.",
     "Write a product headline for this landing page.",
-    "Summarize this random note."
+    "Summarize this random note.",
+    "Just explain what a skill is.",
+    "Draft a blog title for this article.",
+    "Translate this README into Japanese.",
+    "Give me ideas for improving our process."
   ],
   "near_neighbor": [
     "Create a one-off prompt for this task.",
     "Help me brainstorm process ideas without building a skill.",
-    "Improve this README but do not turn it into a skill."
+    "Improve this README but do not turn it into a skill.",
+    "Make a checklist for this task, but not a reusable skill.",
+    "Review this process note and explain it, no packaging needed.",
+    "Help me shape an idea before we decide whether to build a skill.",
+    "Write a custom answer for this request without creating a skill package."
   ]
 }

examples/team-frontend-review/generated-skill/SKILL.md

@@ -5,6 +5,8 @@ description: Review frontend code for accessibility, risky UI security issues, m
 
 # Frontend Review
 
+Review UI code with a pre-merge mindset. Findings come first. Focus on behavior regressions, accessibility gaps, risky form handling, unsafe rendering, and missing user states.
+
 ## Workflow
 
 1. Identify the UI surface and user flows affected.
@@ -12,6 +14,13 @@ description: Review frontend code for accessibility, risky UI security issues, m
 3. Report findings by severity with concrete code references.
 4. Prefer actionable fixes over generic advice.
 
+## Output Contract
+
+- Start with findings, ordered by severity.
+- Include file references for each finding.
+- Call out open questions separately from confirmed issues.
+- If there are no findings, say that explicitly and note residual risks or test gaps.
+
 ## Reference Map
 
 - Read `references/checklist.md` for the review standard.

examples/team-frontend-review/generated-skill/references/checklist.md

@@ -6,3 +6,8 @@
 - Loading states
 - Error states
 - Empty states
+- Focus handling after dialog open/close
+- Color contrast and non-text cues
+- Dangerous HTML injection paths
+- Missing disabled and pending states for forms
+- Inconsistent error recovery or retry flow

scripts/cross_packager.py

@@ -4,53 +4,23 @@ import json
 import shutil
 import zipfile
 from pathlib import Path
+import yaml
 
 
 def read_simple_yaml(path: Path) -> dict:
-    lines = path.read_text(encoding="utf-8").splitlines()
-    data: dict = {}
-    stack: list[tuple[int, dict]] = [(0, data)]
-    for raw_line in lines:
-        if not raw_line.strip() or raw_line.lstrip().startswith("#"):
-            continue
-        indent = len(raw_line) - len(raw_line.lstrip(" "))
-        line = raw_line.strip()
-        while len(stack) > 1 and indent <= stack[-1][0]:
-            stack.pop()
-        parent = stack[-1][1]
-        if line.startswith("- "):
-            item = line[2:].strip().strip("'\"")
-            existing = parent.setdefault("__list__", [])
-            existing.append(item)
-            continue
-        if ":" not in line:
-            continue
-        key, value = line.split(":", 1)
-        key = key.strip()
-        value = value.strip()
-        if value == "":
-            child: dict = {}
-            parent[key] = child
-            stack.append((indent, child))
-        else:
-            parent[key] = value.strip("'\"")
-    return data
+    return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
 
 
 def read_frontmatter(skill_md: Path) -> dict:
+    if not skill_md.exists():
+        raise FileNotFoundError(f"Missing required file: {skill_md}")
     text = skill_md.read_text(encoding="utf-8")
     if not text.startswith("---"):
         return {}
     parts = text.split("---", 2)
     if len(parts) < 3:
         return {}
-    data = {}
-    for line in parts[1].splitlines():
-        if ":" not in line:
-            continue
-        key, value = line.split(":", 1)
-        data[key.strip()] = value.strip().strip("'\"")
-    return data
+    return yaml.safe_load(parts[1]) or {}
 
 
 def read_interface(skill_dir: Path) -> dict:
@@ -58,18 +28,22 @@ def read_interface(skill_dir: Path) -> dict:
     if not path.exists():
         return {}
     raw = read_simple_yaml(path)
-    compatibility = raw.get("compatibility", {})
-    targets = compatibility.get("adapter_targets", {})
-    if isinstance(targets, dict) and "__list__" in targets:
-        compatibility["adapter_targets"] = targets["__list__"]
-    raw["compatibility"] = compatibility
     return raw
 
 
+def require_fields(payload: dict, fields: list[str], label: str) -> None:
+    missing = [field for field in fields if not payload.get(field)]
+    if missing:
+        raise ValueError(f"Missing required {label} fields: {', '.join(missing)}")
+
+
 def build_manifest(skill_dir: Path, platform: str) -> dict:
     frontmatter = read_frontmatter(skill_dir / "SKILL.md")
-    interface = read_interface(skill_dir).get("interface", {})
-    compatibility = read_interface(skill_dir).get("compatibility", {})
+    interface_doc = read_interface(skill_dir)
+    interface = interface_doc.get("interface", {})
+    compatibility = interface_doc.get("compatibility", {})
+    require_fields(frontmatter, ["name", "description"], "frontmatter")
+    require_fields(interface, ["display_name", "short_description", "default_prompt"], "interface")
     return {
         "name": frontmatter.get("name", skill_dir.name),
         "description": frontmatter.get("description", ""),
@@ -127,6 +101,8 @@ def write_yaml_like(path: Path, payload: dict) -> None:
 def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path:
     target_dir = out_dir / "targets" / platform
     target_dir.mkdir(parents=True, exist_ok=True)
+    if platform not in PLATFORM_CONTRACTS:
+        raise ValueError(f"Unsupported platform: {platform}")
     payload = build_manifest(skill_dir, platform)
     if platform == "openai":
         meta_dir = target_dir / "agents"
@@ -223,13 +199,17 @@ def main() -> None:
 
     manifest = copy_manifest(skill_dir, out_dir)
     generated = [str(manifest)]
-    for platform in (args.platform or ["generic"]):
-        generated.append(str(write_adapter(skill_dir, out_dir, platform)))
-    if args.zip:
-        generated.append(str(make_zip(skill_dir, out_dir)))
+    failures = []
+    try:
+        for platform in (args.platform or ["generic"]):
+            generated.append(str(write_adapter(skill_dir, out_dir, platform)))
+        if args.zip:
+            generated.append(str(make_zip(skill_dir, out_dir)))
+    except (FileNotFoundError, ValueError, yaml.YAMLError) as exc:
+        failures.append(str(exc))
 
     expectations = load_expectations(Path(args.expectations).resolve()) if args.expectations else {}
-    validation = validate_exports(out_dir, expectations) if expectations else None
+    validation = validate_exports(out_dir, expectations) if expectations and not failures else None
     report = {
         "output_dir": str(out_dir),
         "generated": generated,
@@ -238,10 +218,13 @@ def main() -> None:
         "failure_handling": {
             "missing_required_file": "exit with code 2 when expectations are provided and validation fails",
             "missing_required_field": "exit with code 2 when expectations are provided and validation fails",
+            "invalid_yaml_or_frontmatter": "exit with code 2 when parsing fails",
+            "unsupported_platform": "exit with code 2 when the platform is not defined in PLATFORM_CONTRACTS",
         },
+        "failures": failures,
     }
     print(json.dumps(report, ensure_ascii=False, indent=2))
-    if validation and not validation["ok"]:
+    if failures or (validation and not validation["ok"]):
         raise SystemExit(2)
 
 

scripts/trigger_eval.py

@@ -3,6 +3,7 @@ import argparse
 import json
 import re
 from pathlib import Path
+from collections import Counter
 
 
 WORD_RE = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_-]*")
@@ -37,12 +38,67 @@ def score_prompt(description_words: set[str], prompt: str) -> float:
     return len(overlap) / len(prompt_words)
 
 
+def token_frequencies(cases: dict, buckets: tuple[str, ...]) -> Counter:
+    freq: Counter = Counter()
+    for bucket in buckets:
+        for prompt in cases.get(bucket, []):
+            freq.update(words(prompt))
+    return freq
+
+
+def compile_negative_patterns(cases: dict) -> list[re.Pattern[str]]:
+    return [re.compile(pattern, re.IGNORECASE) for pattern in cases.get("negative_patterns", [])]
+
+
+def score_prompt_weighted(description_words: set[str], prompt: str, positive_freq: Counter, negative_freq: Counter, negative_patterns: list[re.Pattern[str]]) -> tuple[float, dict]:
+    prompt_words = words(prompt)
+    if not prompt_words:
+        return 0.0, {"matched_positive_tokens": [], "matched_negative_tokens": [], "matched_negative_patterns": []}
+
+    overlap = description_words & prompt_words
+    base_score = len(overlap) / len(prompt_words)
+
+    weighted_bonus = 0.0
+    matched_positive_tokens = []
+    matched_negative_tokens = []
+    for token in overlap:
+        pos = positive_freq.get(token, 0)
+        neg = negative_freq.get(token, 0)
+        if pos > neg:
+            weighted_bonus += 0.06
+            matched_positive_tokens.append(token)
+
+    weighted_penalty = 0.0
+    for token in prompt_words:
+        neg = negative_freq.get(token, 0)
+        pos = positive_freq.get(token, 0)
+        if neg > pos and token not in overlap:
+            weighted_penalty += 0.04
+            matched_negative_tokens.append(token)
+
+    matched_negative_patterns = [pattern.pattern for pattern in negative_patterns if pattern.search(prompt)]
+    pattern_penalty = 0.18 * len(matched_negative_patterns)
+
+    score = max(0.0, min(1.0, base_score + weighted_bonus - weighted_penalty - pattern_penalty))
+    return score, {
+        "matched_positive_tokens": sorted(set(matched_positive_tokens)),
+        "matched_negative_tokens": sorted(set(matched_negative_tokens)),
+        "matched_negative_patterns": matched_negative_patterns,
+        "base_score": round(base_score, 3),
+        "weighted_bonus": round(weighted_bonus, 3),
+        "weighted_penalty": round(weighted_penalty + pattern_penalty, 3),
+    }
+
+
 def classify_bucket(bucket: str) -> bool:
     return bucket == "should_trigger"
 
 
 def evaluate(description: str, cases: dict, threshold: float) -> dict:
     desc_words = words(description)
+    positive_freq = token_frequencies(cases, ("should_trigger",))
+    negative_freq = token_frequencies(cases, ("should_not_trigger", "near_neighbor"))
+    negative_patterns = compile_negative_patterns(cases)
     results = {"should_trigger": [], "should_not_trigger": [], "near_neighbor": []}
     fp = 0
     fn = 0
@@ -54,7 +110,7 @@ def evaluate(description: str, cases: dict, threshold: float) -> dict:
         total = 0
         passed_count = 0
         for prompt in cases.get(bucket, []):
-            score = score_prompt(desc_words, prompt)
+            score, score_detail = score_prompt_weighted(desc_words, prompt, positive_freq, negative_freq, negative_patterns)
             predicted = score >= threshold
             passed = predicted == expected
             total += 1
@@ -70,6 +126,7 @@ def evaluate(description: str, cases: dict, threshold: float) -> dict:
                 "predicted_trigger": predicted,
                 "expected_trigger": expected,
                 "passed": passed,
+                "score_detail": score_detail,
             }
             if 0.75 * threshold <= score <= 1.25 * threshold:
                 record["boundary_case"] = True
@@ -81,6 +138,7 @@ def evaluate(description: str, cases: dict, threshold: float) -> dict:
                         "prompt": prompt,
                         "score": round(score, 3),
                         "reason": "false_negative" if expected else "false_positive",
+                        "matched_negative_patterns": score_detail["matched_negative_patterns"],
                     }
                 )
         bucket_stats[bucket] = {
@@ -95,7 +153,7 @@ def evaluate(description: str, cases: dict, threshold: float) -> dict:
 
     return {
         "threshold": threshold,
-        "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.",
+        "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Final scores combine token overlap, positive-token bonuses, negative-token penalties, and explicit negative-pattern penalties. Scores near the threshold should be reviewed as boundary cases.",
         "false_positives": fp,
         "false_negatives": fn,
         "precision": round(precision, 3) if precision is not None else None,
@@ -128,7 +186,7 @@ def main() -> None:
     parser.add_argument("--baseline-description", help="Baseline description string to compare against")
     parser.add_argument("--baseline-description-file", help="Read baseline description from file")
     parser.add_argument("--cases", required=True, help="JSON file with should_trigger and should_not_trigger arrays")
-    parser.add_argument("--threshold", type=float, default=0.18, help="Token overlap threshold")
+    parser.add_argument("--threshold", type=float, default=None, help="Trigger threshold override")
     args = parser.parse_args()
 
     description = args.description
@@ -138,13 +196,14 @@ def main() -> None:
         raise SystemExit("Provide --description or --description-file")
 
     cases = load_cases(Path(args.cases))
-    report = evaluate(description, cases, args.threshold)
+    threshold = args.threshold if args.threshold is not None else cases.get("recommended_threshold", 0.35)
+    report = evaluate(description, cases, threshold)
 
     baseline = args.baseline_description
     if args.baseline_description_file:
         baseline = extract_description(Path(args.baseline_description_file).read_text(encoding="utf-8"))
     if baseline:
-        report["comparison"] = compare_reports(evaluate(baseline, cases, args.threshold), report)
+        report["comparison"] = compare_reports(evaluate(baseline, cases, threshold), report)
 
     print(json.dumps(report, ensure_ascii=False, indent=2))
     if report["false_positives"] > 2: