diff --git a/.gitignore b/.gitignore index f003499..6da79bf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ tests/tmp_iteration_directions/ tests/tmp_review_viewer/ tests/tmp_feedback/ tests/tmp_github_benchmark_scan/ +tests/tmp_intent_confidence/ +tests/tmp_reference_synthesis/ reports/release_snapshots/ diff --git a/Makefile b/Makefile index e9adb4c..e4e1699 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -47,6 +47,12 @@ reference-scan-check: github-benchmark-scan-check: $(PYTHON) tests/verify_github_benchmark_scan.py +intent-confidence-check: + $(PYTHON) tests/verify_intent_confidence.py + +reference-synthesis-check: + $(PYTHON) tests/verify_reference_synthesis.py + iteration-directions-check: $(PYTHON) tests/verify_iteration_directions.py @@ -98,10 +104,10 @@ resource-boundary-check: quality-check: $(PYTHON) tests/verify_quality_checks.py -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md diff --git a/reports/context_budget.json b/reports/context_budget.json index 527fce3..1f6abf0 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 824, - "other_text_tokens": 277008, + "other_text_tokens": 291352, "estimated_initial_load_tokens": 1000, - "estimated_total_text_tokens": 277832, - "relevant_file_count": 167, + "estimated_total_text_tokens": 292176, + "relevant_file_count": 184, "unused_resource_dirs": [], "quality_signal_points": 140, "quality_density": 140.0 diff --git a/reports/intent-confidence.json b/reports/intent-confidence.json new file mode 100644 index 0000000..a6d252b --- /dev/null +++ b/reports/intent-confidence.json @@ -0,0 +1,79 @@ +{ + "score": 30, + "band": "low", + "gate_passed": false, + "strengths": [ + "The recurring job is concrete enough to anchor the package." + ], + "gaps": [ + { + "key": "real_inputs", + "label": "Real inputs are missing", + "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", + "severity": "high" + }, + { + "key": "primary_output", + "label": "Primary output is missing", + "reason": "The package does not yet know what it must hand back.", + "severity": "high" + }, + { + "key": "exclusions", + "label": "Near-neighbor exclusions are missing", + "reason": "The route may blur into nearby requests without an exclusion list.", + "severity": "high" + }, + { + "key": "constraints", + "label": "Constraints are missing", + "reason": "The package does not yet know which tradeoffs matter most.", + "severity": "high" + }, + { + "key": "standards", + "label": "Quality bar is implied, not explicit", + "reason": "The first evaluation target is still underspecified.", + "severity": "medium" + } + ], + "follow_up_questions": [ + { + "slot": "real_inputs", + "question": "What material will people actually hand to this skill in practice?", + "why": "Real input shape decides whether references, scripts, or examples are needed.", + "list": true, + "label": "Real inputs are missing", + "severity": "high" + }, + { + "slot": "primary_output", + "question": "What finished hand-back should this skill return so the next person can keep moving?", + "why": "The output is the anchor for package design and review.", + "list": false, + "label": "Primary output is missing", + "severity": "high" + }, + { + "slot": "exclusions", + "question": "What nearby requests should this skill clearly leave out so the boundary stays clean?", + "why": "Exclusions are the fastest route to better trigger quality.", + "list": true, + "label": "Near-neighbor exclusions are missing", + "severity": "high" + } + ], + "anchor_sentence": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "context": { + "job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "real_inputs": [], + "primary_output": "", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [] + } +} diff --git a/reports/intent-confidence.md b/reports/intent-confidence.md new file mode 100644 index 0000000..07c3a24 --- /dev/null +++ b/reports/intent-confidence.md @@ -0,0 +1,31 @@ +# Intent Confidence + +- Confidence score: `30/100` +- Confidence band: `low` +- Gate passed: `False` +- Recommended action: Pause before deep authoring and close the highest-leverage gaps first. + +## Current Reading + +Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. + +## Strong Signals + +- The recurring job is concrete enough to anchor the package. + +## Gaps To Close + +- **Real inputs are missing** (`high`): Without real inputs, it is hard to choose assets, scripts, or examples. +- **Primary output is missing** (`high`): The package does not yet know what it must hand back. +- **Near-neighbor exclusions are missing** (`high`): The route may blur into nearby requests without an exclusion list. +- **Constraints are missing** (`high`): The package does not yet know which tradeoffs matter most. +- **Quality bar is implied, not explicit** (`medium`): The first evaluation target is still underspecified. + +## Follow-Up Questions + +- **What material will people actually hand to this skill in practice?** + - Why: Real input shape decides whether references, scripts, or examples are needed. +- **What finished hand-back should this skill return so the next person can keep moving?** + - Why: The output is the anchor for package design and review. +- **What nearby requests should this skill clearly leave out so the boundary stays clean?** + - Why: Exclusions are the fastest route to better trigger quality. diff --git a/reports/intent-context.json b/reports/intent-context.json new file mode 100644 index 0000000..8fc3e49 --- /dev/null +++ b/reports/intent-context.json @@ -0,0 +1,11 @@ +{ + "job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "real_inputs": [], + "primary_output": "", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [] +} diff --git a/reports/reference-synthesis.json b/reports/reference-synthesis.json new file mode 100644 index 0000000..4de1d73 --- /dev/null +++ b/reports/reference-synthesis.json @@ -0,0 +1,69 @@ +{ + "skill_name": "yao-meta-skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "intent_confidence": { + "score": 30, + "band": "low", + "gate_passed": false + }, + "github_benchmarks": [], + "source_tracks": [ + { + "source_type": "official", + "name": "Official workflow product ergonomics", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "workflow" + ], + "borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "avoid": "Do not mimic product polish that adds UI bulk without improving clarity.", + "why_relevant": "This track matches: workflow." + }, + { + "source_type": "research", + "name": "Hypothesis-test-learn loop", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", + "why_relevant": "This track matches: general fit." + }, + { + "source_type": "principles", + "name": "Boundary-first design", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", + "why_relevant": "This track matches: general fit." + } + ], + "synthesis": { + "borrow_now": [ + "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "avoid_now": [ + "Do not mimic product polish that adds UI bulk without improving clarity.", + "Do not create experimental overhead that exceeds the skill's real risk tier.", + "Do not expand execution assets until route boundaries stay clean." + ], + "quality_risers": [ + "Use GitHub repositories for concrete package and workflow patterns.", + "Use curated official or commercial tracks for entrypoint and operator ergonomics.", + "Use research tracks to justify the smallest evaluation loop that still catches regressions.", + "Use principle tracks to keep the package small, boundary-aware, and outcome-driven." + ], + "decision_prompt": "I pulled concrete GitHub benchmarks and layered them with curated official, research, and principle tracks. Do you want the next draft to borrow one or two of these patterns now, or keep the first pass lighter?", + "source_mix": { + "github_benchmarks": 0, + "curated_tracks": 3, + "user_references": 0 + } + } +} diff --git a/reports/reference-synthesis.md b/reports/reference-synthesis.md new file mode 100644 index 0000000..1829c9c --- /dev/null +++ b/reports/reference-synthesis.md @@ -0,0 +1,55 @@ +# Reference Synthesis + +Skill: `yao-meta-skill` +- Description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +- Intent confidence: `30/100` (`low`) + +## Live GitHub Benchmarks + +- No live GitHub benchmarks are attached yet. + +## Curated World-Class Pattern Tracks + +### Official workflow product ergonomics +- Type: `official` +- Evidence mode: `curated-pattern-track` +- Why relevant: This track matches: workflow. +- Borrow: Borrow a first-time operator flow that explains itself before it asks for more structure. +- Avoid: Do not mimic product polish that adds UI bulk without improving clarity. + +### Hypothesis-test-learn loop +- Type: `research` +- Evidence mode: `curated-pattern-track` +- Why relevant: This track matches: general fit. +- Borrow: Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. +- Avoid: Do not create experimental overhead that exceeds the skill's real risk tier. + +### Boundary-first design +- Type: `principles` +- Evidence mode: `curated-pattern-track` +- Why relevant: This track matches: general fit. +- Borrow: Borrow the discipline of defining what the skill should not own before growing the package. +- Avoid: Do not expand execution assets until route boundaries stay clean. + +## Borrow Now + +- Borrow a first-time operator flow that explains itself before it asks for more structure. +- Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. +- Borrow the discipline of defining what the skill should not own before growing the package. + +## Avoid Now + +- Do not mimic product polish that adds UI bulk without improving clarity. +- Do not create experimental overhead that exceeds the skill's real risk tier. +- Do not expand execution assets until route boundaries stay clean. + +## Quality Lift Thesis + +- Use GitHub repositories for concrete package and workflow patterns. +- Use curated official or commercial tracks for entrypoint and operator ergonomics. +- Use research tracks to justify the smallest evaluation loop that still catches regressions. +- Use principle tracks to keep the package small, boundary-aware, and outcome-driven. + +## Decision Prompt + +I pulled concrete GitHub benchmarks and layered them with curated official, research, and principle tracks. Do you want the next draft to borrow one or two of these patterns now, or keep the first pass lighter? diff --git a/scripts/init_skill.py b/scripts/init_skill.py index 1f5b634..de282b1 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -5,9 +5,11 @@ from datetime import date from pathlib import Path from github_benchmark_scan import run_github_benchmark_scan +from render_intent_confidence import render_intent_confidence from render_intent_dialogue import render_intent_dialogue from render_iteration_directions import render_iteration_directions from render_reference_scan import parse_reference, render_reference_scan +from render_reference_synthesis import render_reference_synthesis from render_review_viewer import render_review_viewer from render_skill_overview import render_skill_overview @@ -40,10 +42,12 @@ README_TEMPLATE = """# {title} 1. Load the skill through `SKILL.md`. 2. Start with `reports/intent-dialogue.md` to tighten the real job, outputs, exclusions, and the standards you care about. 3. Open `reports/reference-scan.md` to capture external benchmarks and any user-supplied references worth learning from. -4. Follow the workflow steps in `SKILL.md`. -5. Check `reports/skill-overview.html` if you want a fast visual explanation of the package. -6. Open `reports/review-viewer.html` for a compact visual review of the package. -7. Review `reports/iteration-directions.md` for the three most valuable next moves. +4. Review `reports/intent-confidence.md` to see whether the real job, inputs, outputs, and exclusions are clear enough yet. +5. Open `reports/reference-synthesis.md` to see the GitHub benchmarks plus curated official, research, and principle tracks in one place. +6. Follow the workflow steps in `SKILL.md`. +7. Check `reports/skill-overview.html` if you want a fast visual explanation of the package. +8. Open `reports/review-viewer.html` for a compact visual review of the package. +9. Review `reports/iteration-directions.md` for the three most valuable next moves. ## Package Map @@ -51,8 +55,10 @@ README_TEMPLATE = """# {title} - `agents/interface.yaml`: portable interface metadata - `manifest.json`: lifecycle and packaging metadata - `reports/intent-dialogue.md`: front-loaded discovery questions for better boundary design and clearer human alignment +- `reports/intent-confidence.md`: current clarity score, open gaps, and the next follow-up questions worth asking - `reports/github-benchmark-scan.md`: top public benchmark repositories, extracted patterns, and borrow or avoid notes - `reports/reference-scan.md`: benchmark notes from public references, user references, and local constraints +- `reports/reference-synthesis.md`: a combined view of GitHub benchmarks plus curated world-class pattern tracks - `reports/skill-overview.html`: visual overview report - `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps - `reports/iteration-directions.md`: the top three next iteration directions @@ -172,6 +178,7 @@ def initialize_skill( github_query: str | None = None, github_top_n: int = 3, github_fixture_dir: str | None = None, + intent_context: dict | None = None, ) -> dict: title = title or name.replace("-", " ").title() root = Path(output_dir).resolve() / name @@ -200,7 +207,7 @@ def initialize_skill( json.dumps(build_manifest(name, mode, archetype), ensure_ascii=False, indent=2), encoding="utf-8", ) - overview = render_skill_overview(root) + intent_confidence = render_intent_confidence(root, context=intent_context) intent_dialogue = render_intent_dialogue(root) benchmark_scan = None combined_external_references = [*(external_references or [])] @@ -216,17 +223,24 @@ def initialize_skill( root, dedupe_references([*combined_external_references, *(user_references or []), *(local_constraints or [])]), ) + reference_synthesis = render_reference_synthesis(root) + overview = render_skill_overview(root) iteration_directions = render_iteration_directions(root) review_viewer = render_review_viewer(root) artifacts = { "readme": str(root / "README.md"), "manifest": str(root / "manifest.json"), + "intent_confidence_md": intent_confidence["artifacts"]["markdown"], + "intent_confidence_json": intent_confidence["artifacts"]["json"], + "intent_context_json": intent_confidence["artifacts"]["context_json"], "skill_overview_html": overview["artifacts"]["html"], "skill_overview_json": overview["artifacts"]["json"], "intent_dialogue_md": intent_dialogue["artifacts"]["markdown"], "intent_dialogue_json": intent_dialogue["artifacts"]["json"], "reference_scan_md": reference_scan["artifacts"]["markdown"], "reference_scan_json": reference_scan["artifacts"]["json"], + "reference_synthesis_md": reference_synthesis["artifacts"]["markdown"], + "reference_synthesis_json": reference_synthesis["artifacts"]["json"], "iteration_directions_md": iteration_directions["artifacts"]["markdown"], "iteration_directions_json": iteration_directions["artifacts"]["json"], "review_viewer_html": review_viewer["artifacts"]["html"], @@ -241,6 +255,8 @@ def initialize_skill( "mode": mode, "archetype": archetype, "github_benchmark_scan": benchmark_scan, + "intent_confidence": intent_confidence["summary"], + "reference_synthesis": reference_synthesis["summary"], "artifacts": artifacts, } @@ -259,6 +275,13 @@ def main() -> None: parser.add_argument("--github-query") parser.add_argument("--github-top-n", type=int, default=3) parser.add_argument("--github-fixture-dir") + parser.add_argument("--intent-job") + parser.add_argument("--intent-real-input", action="append", default=[]) + parser.add_argument("--intent-primary-output") + parser.add_argument("--intent-exclusion", action="append", default=[]) + parser.add_argument("--intent-constraint", action="append", default=[]) + parser.add_argument("--intent-standard", action="append", default=[]) + parser.add_argument("--intent-correction", default="") args = parser.parse_args() result = initialize_skill( args.name, @@ -273,6 +296,17 @@ def main() -> None: github_query=args.github_query, github_top_n=args.github_top_n, github_fixture_dir=args.github_fixture_dir, + intent_context={ + "job": args.intent_job or args.description, + "real_inputs": args.intent_real_input, + "primary_output": args.intent_primary_output or "", + "description": args.description, + "exclusions": args.intent_exclusion, + "constraints": args.intent_constraint, + "standards": args.intent_standard, + "correction": args.intent_correction, + "user_references": [parse_reference(item, "user")["name"] for item in args.user_reference], + }, ) print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/scripts/render_intent_confidence.py b/scripts/render_intent_confidence.py new file mode 100644 index 0000000..007b1cb --- /dev/null +++ b/scripts/render_intent_confidence.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +GENERIC_PHRASES = { + "turn a repeated workflow into a reusable skill", + "a reusable skill package", + "describe what the skill does and when to use it", + "turn rough requests into a compact reusable demo skill", +} + +GENERIC_TOKENS = { + "workflow", + "skill", + "package", + "reusable", + "repeated", + "request", + "requests", + "task", + "tasks", + "work", + "job", +} + +FOLLOW_UP_LIBRARY = { + "job_specificity": { + "slot": "job", + "question": "If you say it plainly, what concrete repeated task should this skill own every time?", + "why": "A skill needs a real recurring job, not only a generic packaging goal.", + "list": False, + }, + "real_inputs": { + "slot": "real_inputs", + "question": "What material will people actually hand to this skill in practice?", + "why": "Real input shape decides whether references, scripts, or examples are needed.", + "list": True, + }, + "primary_output": { + "slot": "primary_output", + "question": "What finished hand-back should this skill return so the next person can keep moving?", + "why": "The output is the anchor for package design and review.", + "list": False, + }, + "exclusions": { + "slot": "exclusions", + "question": "What nearby requests should this skill clearly leave out so the boundary stays clean?", + "why": "Exclusions are the fastest route to better trigger quality.", + "list": True, + }, + "constraints": { + "slot": "constraints", + "question": "What constraints matter most here: privacy, naming, compatibility, portability, governance, or speed?", + "why": "Constraints decide how much structure and validation this skill really needs.", + "list": True, + }, + "standards": { + "slot": "standards", + "question": "What quality bar matters most here: consistency, auditability, tone, or delivery speed?", + "why": "Standards explain how to choose the first evaluation gate.", + "list": True, + }, +} + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {}, text + frontmatter_text = "\n".join(lines[1:end_index]) + body = "\n".join(lines[end_index + 1 :]).lstrip() + if yaml is not None: + payload = yaml.safe_load(frontmatter_text) or {} + return payload if isinstance(payload, dict) else {}, body + data = {} + for line in frontmatter_text.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data, body + + +def normalized_list(value: list[str] | str | None) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + parts = [item.strip() for item in value.split(",")] + return [item for item in parts if item] + return [str(item).strip() for item in value if str(item).strip()] + + +def tokenize(text: str) -> list[str]: + return re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text.lower()) + + +def is_generic(text: str) -> bool: + compact = " ".join(text.lower().split()) + if not compact: + return True + if compact in GENERIC_PHRASES: + return True + tokens = tokenize(compact) + if len(tokens) <= 3: + return True + content_tokens = [token for token in tokens if token not in GENERIC_TOKENS] + return len(content_tokens) < 2 + + +def build_context_from_skill(skill_dir: Path) -> dict[str, Any]: + skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8") + frontmatter, _ = parse_frontmatter(skill_text) + payload = load_json(skill_dir / "reports" / "intent-context.json") + if payload: + return payload + return { + "job": frontmatter.get("description", ""), + "real_inputs": [], + "primary_output": "", + "description": frontmatter.get("description", ""), + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [], + } + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + payload = json.loads(path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else {} + + +def assess_intent_confidence(context: dict[str, Any]) -> dict[str, Any]: + job = str(context.get("job", "")).strip() + primary_output = str(context.get("primary_output", "")).strip() + description = str(context.get("description", "")).strip() + real_inputs = normalized_list(context.get("real_inputs")) + exclusions = normalized_list(context.get("exclusions")) + constraints = normalized_list(context.get("constraints")) + standards = normalized_list(context.get("standards")) + user_references = normalized_list(context.get("user_references")) + correction = str(context.get("correction", "")).strip() + + score = 0 + strengths = [] + gaps = [] + + def add_gap(key: str, label: str, reason: str, severity: str = "high") -> None: + gaps.append({"key": key, "label": label, "reason": reason, "severity": severity}) + + if job and not is_generic(job): + score += 25 + strengths.append("The recurring job is concrete enough to anchor the package.") + elif job: + score += 10 + add_gap( + "job_specificity", + "Recurring job is still generic", + "The current job statement sounds more like a packaging goal than a concrete repeated task.", + ) + else: + add_gap("job_specificity", "Recurring job is missing", "The package has no clear job-to-be-done anchor yet.") + + if real_inputs: + score += 15 + strengths.append("Real input shape is explicit.") + else: + add_gap("real_inputs", "Real inputs are missing", "Without real inputs, it is hard to choose assets, scripts, or examples.") + + if primary_output and not is_generic(primary_output): + score += 20 + strengths.append("The hand-back output is concrete.") + elif primary_output: + score += 8 + add_gap( + "primary_output", + "Primary output is still generic", + "The current output does not yet say what a useful finished deliverable looks like.", + ) + else: + add_gap("primary_output", "Primary output is missing", "The package does not yet know what it must hand back.") + + if exclusions: + score += 15 + strengths.append("Boundary exclusions are already explicit.") + else: + add_gap("exclusions", "Near-neighbor exclusions are missing", "The route may blur into nearby requests without an exclusion list.") + + if constraints: + score += 10 + strengths.append("Operational constraints are visible.") + else: + add_gap("constraints", "Constraints are missing", "The package does not yet know which tradeoffs matter most.") + + if standards: + score += 5 + strengths.append("Quality standards are visible.") + else: + add_gap("standards", "Quality bar is implied, not explicit", "The first evaluation target is still underspecified.", "medium") + + if correction: + score += 5 + strengths.append("A correction loop already tightened the first reading.") + + if user_references: + score += 5 + strengths.append("Reference preferences are already available.") + + if description and not is_generic(description): + score += 5 + + score = min(score, 100) + if score >= 85: + band = "high" + elif score >= 70: + band = "medium" + else: + band = "low" + + gate_passed = score >= 70 and not any( + gap["key"] in {"job_specificity", "real_inputs", "primary_output"} and gap["severity"] == "high" + for gap in gaps + ) + follow_up_questions = [ + { + **FOLLOW_UP_LIBRARY[gap["key"]], + "label": gap["label"], + "severity": gap["severity"], + } + for gap in gaps + if gap["key"] in FOLLOW_UP_LIBRARY + ][:3] + + anchor_sentence = " ".join( + item + for item in [ + job or "Unclear recurring job.", + f"Primary output: {primary_output}." if primary_output else "", + f"Exclusions: {', '.join(exclusions)}." if exclusions else "", + ] + if item + ).strip() + + return { + "score": score, + "band": band, + "gate_passed": gate_passed, + "strengths": strengths[:5], + "gaps": gaps, + "follow_up_questions": follow_up_questions, + "anchor_sentence": anchor_sentence, + "recommended_action": ( + "Intent is clear enough to package the first routeable version." + if gate_passed + else "Pause before deep authoring and close the highest-leverage gaps first." + ), + "context": { + "job": job, + "real_inputs": real_inputs, + "primary_output": primary_output, + "description": description, + "exclusions": exclusions, + "constraints": constraints, + "standards": standards, + "correction": correction, + "user_references": user_references, + }, + } + + +def render_markdown(summary: dict[str, Any]) -> str: + lines = [ + "# Intent Confidence", + "", + f"- Confidence score: `{summary['score']}/100`", + f"- Confidence band: `{summary['band']}`", + f"- Gate passed: `{summary['gate_passed']}`", + f"- Recommended action: {summary['recommended_action']}", + "", + "## Current Reading", + "", + summary["anchor_sentence"] or "No clear anchor sentence yet.", + "", + "## Strong Signals", + "", + ] + if summary["strengths"]: + for item in summary["strengths"]: + lines.append(f"- {item}") + else: + lines.append("- No strong signals yet.") + + lines.extend(["", "## Gaps To Close", ""]) + if summary["gaps"]: + for gap in summary["gaps"]: + lines.append(f"- **{gap['label']}** (`{gap['severity']}`): {gap['reason']}") + else: + lines.append("- No major intent gaps detected.") + + lines.extend(["", "## Follow-Up Questions", ""]) + if summary["follow_up_questions"]: + for item in summary["follow_up_questions"]: + lines.append(f"- **{item['question']}**") + lines.append(f" - Why: {item['why']}") + else: + lines.append("- No extra follow-up questions required before the first package.") + return "\n".join(lines).strip() + "\n" + + +def render_intent_confidence( + skill_dir: Path, + context: dict[str, Any] | None = None, + output_md: Path | None = None, + output_json: Path | None = None, + context_json: Path | None = None, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + output_md = output_md or reports_dir / "intent-confidence.md" + output_json = output_json or reports_dir / "intent-confidence.json" + context_json = context_json or reports_dir / "intent-context.json" + + context_payload = context or build_context_from_skill(skill_dir) + summary = assess_intent_confidence(context_payload) + output_md.write_text(render_markdown(summary), encoding="utf-8") + output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + context_json.write_text(json.dumps(summary["context"], ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return { + "ok": True, + "skill_dir": str(skill_dir), + "artifacts": { + "markdown": str(output_md), + "json": str(output_json), + "context_json": str(context_json), + }, + "summary": summary, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render an intent confidence report for a skill package.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--context-json") + parser.add_argument("--output-md") + parser.add_argument("--output-json") + args = parser.parse_args() + + context = None + if args.context_json: + context = load_json(Path(args.context_json).resolve()) + + result = render_intent_confidence( + Path(args.skill_dir), + context=context, + output_md=Path(args.output_md).resolve() if args.output_md else None, + output_json=Path(args.output_json).resolve() if args.output_json else None, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_reference_synthesis.py b/scripts/render_reference_synthesis.py new file mode 100644 index 0000000..40c84fb --- /dev/null +++ b/scripts/render_reference_synthesis.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +CURATED_TRACKS = [ + { + "source_type": "official", + "name": "Official skill anatomy and context discipline", + "keywords": ["adapter", "portable", "metadata", "description", "references", "context", "entrypoint"], + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + }, + { + "source_type": "official", + "name": "Official workflow product ergonomics", + "keywords": ["quickstart", "review", "viewer", "feedback", "operator", "workflow", "guide"], + "borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "avoid": "Do not mimic product polish that adds UI bulk without improving clarity.", + }, + { + "source_type": "research", + "name": "Hypothesis-test-learn loop", + "keywords": ["test", "benchmark", "baseline", "compare", "holdout", "optimize", "iteration"], + "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", + }, + { + "source_type": "research", + "name": "Human-in-the-loop verification", + "keywords": ["review", "audit", "govern", "incident", "compliance", "approval"], + "borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.", + "avoid": "Do not force every skill through heavyweight review when the risk is low.", + }, + { + "source_type": "principles", + "name": "Boundary-first design", + "keywords": ["route", "trigger", "boundary", "exclude", "scope", "near-neighbor"], + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", + }, + { + "source_type": "principles", + "name": "Minimum sufficient structure", + "keywords": ["lightweight", "lean", "minimal", "small", "context", "scaffold", "focus"], + "borrow": "Borrow the smallest structure that makes the skill reliable and explainable.", + "avoid": "Do not add files or gates that raise context cost faster than they raise trust.", + }, + { + "source_type": "principles", + "name": "Outcome-backwards design", + "keywords": ["output", "deliverable", "result", "handoff", "keep moving", "packet", "summary"], + "borrow": "Borrow the habit of designing from the required hand-back output backwards.", + "avoid": "Do not start with architecture terms before the deliverable is concrete.", + }, +] + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + payload = json.loads(path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else {} + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {}, text + frontmatter_text = "\n".join(lines[1:end_index]) + body = "\n".join(lines[end_index + 1 :]).lstrip() + if yaml is not None: + payload = yaml.safe_load(frontmatter_text) or {} + return payload if isinstance(payload, dict) else {}, body + data = {} + for line in frontmatter_text.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data, body + + +def anchor_text(skill_dir: Path, benchmark: dict[str, Any], intent: dict[str, Any]) -> str: + skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8") + frontmatter, _ = parse_frontmatter(skill_text) + pieces = [ + frontmatter.get("name", skill_dir.name), + frontmatter.get("description", ""), + benchmark.get("query", ""), + intent.get("anchor_sentence", ""), + ] + return " ".join(piece for piece in pieces if piece).lower() + + +def match_keywords(text: str, keywords: list[str]) -> list[str]: + hits = [] + for keyword in keywords: + if keyword in text: + hits.append(keyword) + return hits + + +def select_source_tracks(text: str) -> list[dict[str, Any]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for track in CURATED_TRACKS: + matched = match_keywords(text, track["keywords"]) + score = len(matched) + payload = {**track, "matched_keywords": matched, "score": score} + grouped.setdefault(track["source_type"], []).append(payload) + + selected = [] + for source_type in ("official", "research", "principles"): + candidates = sorted(grouped.get(source_type, []), key=lambda item: item["score"], reverse=True) + chosen = candidates[0] if candidates else None + if chosen is None: + continue + if chosen["score"] == 0: + chosen = {**chosen, "matched_keywords": ["general fit"]} + selected.append( + { + "source_type": source_type, + "name": chosen["name"], + "evidence_mode": "curated-pattern-track", + "matched_keywords": chosen["matched_keywords"], + "borrow": chosen["borrow"], + "avoid": chosen["avoid"], + "why_relevant": ( + f"This track matches: {', '.join(chosen['matched_keywords'])}." + if chosen["matched_keywords"] + else "This track is the best general fit for the current skill shape." + ), + } + ) + return selected + + +def unique_items(items: list[str], limit: int) -> list[str]: + seen = set() + output = [] + for item in items: + if item in seen: + continue + seen.add(item) + output.append(item) + if len(output) == limit: + break + return output + + +def build_summary(skill_dir: Path) -> dict[str, Any]: + skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8") + frontmatter, _ = parse_frontmatter(skill_text) + benchmark = load_json(skill_dir / "reports" / "github-benchmark-scan.json") + intent_payload = load_json(skill_dir / "reports" / "intent-confidence.json") + reference_scan = load_json(skill_dir / "reports" / "reference-scan.json") + + source_tracks = select_source_tracks(anchor_text(skill_dir, benchmark, intent_payload)) + github_repos = benchmark.get("repositories", [])[:3] + github_borrow = benchmark.get("cross_repo", {}).get("borrow", []) + github_avoid = benchmark.get("cross_repo", {}).get("avoid", []) + track_borrow = [track["borrow"] for track in source_tracks] + track_avoid = [track["avoid"] for track in source_tracks] + user_refs = reference_scan.get("user_references", []) + + borrow_now = unique_items( + [ + *track_borrow, + *github_borrow, + *[ref.get("borrow", "") for ref in user_refs], + ], + 5, + ) + avoid_now = unique_items( + [ + *track_avoid, + *github_avoid, + *[ref.get("avoid", "") for ref in user_refs], + ], + 5, + ) + quality_risers = unique_items( + [ + "Use GitHub repositories for concrete package and workflow patterns.", + "Use curated official or commercial tracks for entrypoint and operator ergonomics.", + "Use research tracks to justify the smallest evaluation loop that still catches regressions.", + "Use principle tracks to keep the package small, boundary-aware, and outcome-driven.", + ], + 4, + ) + + return { + "skill_name": frontmatter.get("name", skill_dir.name), + "description": frontmatter.get("description", "No description found."), + "intent_confidence": { + "score": intent_payload.get("score", 0), + "band": intent_payload.get("band", "low"), + "gate_passed": intent_payload.get("gate_passed", False), + }, + "github_benchmarks": [ + { + "name": repo.get("full_name"), + "url": repo.get("html_url"), + "stars": repo.get("stars"), + "borrow": repo.get("borrow", [])[:2], + } + for repo in github_repos + ], + "source_tracks": source_tracks, + "synthesis": { + "borrow_now": borrow_now, + "avoid_now": avoid_now, + "quality_risers": quality_risers, + "decision_prompt": ( + "I pulled concrete GitHub benchmarks and layered them with curated official, research, and principle tracks. " + "Do you want the next draft to borrow one or two of these patterns now, or keep the first pass lighter?" + ), + "source_mix": { + "github_benchmarks": len(github_repos), + "curated_tracks": len(source_tracks), + "user_references": len(user_refs), + }, + }, + } + + +def render_markdown(summary: dict[str, Any]) -> str: + lines = [ + "# Reference Synthesis", + "", + f"Skill: `{summary['skill_name']}`", + f"- Description: {summary['description']}", + f"- Intent confidence: `{summary['intent_confidence']['score']}/100` (`{summary['intent_confidence']['band']}`)", + "", + "## Live GitHub Benchmarks", + "", + ] + if summary["github_benchmarks"]: + for repo in summary["github_benchmarks"]: + lines.extend( + [ + f"### {repo['name']}", + f"- URL: {repo['url']}", + f"- Stars: `{repo['stars']}`", + ] + ) + for item in repo.get("borrow", []): + lines.append(f"- Borrow: {item}") + lines.append("") + else: + lines.append("- No live GitHub benchmarks are attached yet.") + lines.append("") + + lines.extend(["## Curated World-Class Pattern Tracks", ""]) + for track in summary["source_tracks"]: + lines.extend( + [ + f"### {track['name']}", + f"- Type: `{track['source_type']}`", + f"- Evidence mode: `{track['evidence_mode']}`", + f"- Why relevant: {track['why_relevant']}", + f"- Borrow: {track['borrow']}", + f"- Avoid: {track['avoid']}", + "", + ] + ) + + lines.extend(["## Borrow Now", ""]) + for item in summary["synthesis"]["borrow_now"]: + lines.append(f"- {item}") + + lines.extend(["", "## Avoid Now", ""]) + for item in summary["synthesis"]["avoid_now"]: + lines.append(f"- {item}") + + lines.extend(["", "## Quality Lift Thesis", ""]) + for item in summary["synthesis"]["quality_risers"]: + lines.append(f"- {item}") + + lines.extend(["", "## Decision Prompt", "", summary["synthesis"]["decision_prompt"], ""]) + return "\n".join(lines).strip() + "\n" + + +def render_reference_synthesis( + skill_dir: Path, + output_md: Path | None = None, + output_json: Path | None = None, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + output_md = output_md or reports_dir / "reference-synthesis.md" + output_json = output_json or reports_dir / "reference-synthesis.json" + summary = build_summary(skill_dir) + output_md.write_text(render_markdown(summary), encoding="utf-8") + output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return { + "ok": True, + "skill_dir": str(skill_dir), + "artifacts": { + "markdown": str(output_md), + "json": str(output_json), + }, + "summary": summary, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a multi-source reference synthesis report for a skill package.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-md") + parser.add_argument("--output-json") + args = parser.parse_args() + + result = render_reference_synthesis( + Path(args.skill_dir), + output_md=Path(args.output_md).resolve() if args.output_md else None, + output_json=Path(args.output_json).resolve() if args.output_json else None, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_review_viewer.py b/scripts/render_review_viewer.py index 6ee72a7..4c87bac 100644 --- a/scripts/render_review_viewer.py +++ b/scripts/render_review_viewer.py @@ -5,9 +5,11 @@ import json import re from pathlib import Path +from render_intent_confidence import render_intent_confidence from render_intent_dialogue import render_intent_dialogue from render_iteration_directions import render_iteration_directions from render_reference_scan import render_reference_scan +from render_reference_synthesis import render_reference_synthesis from render_skill_overview import render_skill_overview @@ -49,28 +51,41 @@ def load_benchmark_summary(skill_dir: Path) -> dict: return payload if isinstance(payload, dict) else {} +def load_reference_synthesis_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "reference-synthesis.json") + return payload if isinstance(payload, dict) else {} + + def ensure_report_inputs(skill_dir: Path) -> dict: overview_json = skill_dir / "reports" / "skill-overview.json" + intent_confidence_json = skill_dir / "reports" / "intent-confidence.json" intent_json = skill_dir / "reports" / "intent-dialogue.json" reference_json = skill_dir / "reports" / "reference-scan.json" + reference_synthesis_json = skill_dir / "reports" / "reference-synthesis.json" directions_json = skill_dir / "reports" / "iteration-directions.json" overview_payload = load_json(overview_json) if overview_json.exists() else {} + intent_confidence_payload = load_json(intent_confidence_json) if intent_confidence_json.exists() else {} intent_payload = load_json(intent_json) if intent_json.exists() else {} reference_payload = load_json(reference_json) if reference_json.exists() else {} + reference_synthesis_payload = load_json(reference_synthesis_json) if reference_synthesis_json.exists() else {} directions_payload = load_json(directions_json) if directions_json.exists() else {} - overview = overview_payload or render_skill_overview(skill_dir)["summary"] + intent_confidence = intent_confidence_payload or render_intent_confidence(skill_dir)["summary"] intent = intent_payload or render_intent_dialogue(skill_dir)["summary"] reference = reference_payload or render_reference_scan(skill_dir, [])["summary"] + reference_synthesis = reference_synthesis_payload or render_reference_synthesis(skill_dir)["summary"] + overview = overview_payload or render_skill_overview(skill_dir)["summary"] iteration = directions_payload.get("summary", {}) or render_iteration_directions(skill_dir)["summary"] feedback = load_feedback_summary(skill_dir) baseline = load_baseline_summary(skill_dir) compare = load_specific_compare(skill_dir) promotion = load_specific_promotion(skill_dir) benchmark = load_benchmark_summary(skill_dir) + reference_synthesis = load_reference_synthesis_summary(skill_dir) return { "overview": overview, + "intent_confidence": intent_confidence, "intent": intent, "reference": reference, "iteration": directions_payload if directions_payload else {"summary": iteration, "directions": []}, @@ -79,6 +94,7 @@ def ensure_report_inputs(skill_dir: Path) -> dict: "compare": compare, "promotion": promotion, "benchmark": benchmark, + "reference_synthesis": reference_synthesis, } @@ -133,6 +149,19 @@ def benchmark_cards(benchmark: dict) -> list[dict]: return cards +def synthesis_cards(reference_synthesis: dict) -> list[dict]: + cards = [] + for track in reference_synthesis.get("source_tracks", [])[:3]: + cards.append( + { + "name": track.get("name", "Unknown track"), + "borrow": [track.get("borrow", "")] if track.get("borrow") else [], + "avoid": [track.get("avoid", "")] if track.get("avoid") else [], + } + ) + return cards + + def split_sentences(text: str) -> list[str]: if not text: return [] @@ -197,6 +226,7 @@ def variant_diff_cards(compare: dict) -> list[dict]: def render_html(report: dict) -> str: overview = report["overview"] intent = report["intent"] + intent_confidence = report.get("intent_confidence", {}) reference = report["reference"] iteration = report["iteration"] directions = iteration.get("directions", [])[:3] @@ -205,9 +235,11 @@ def render_html(report: dict) -> str: compare = report.get("compare", {}) promotion = report.get("promotion", {}) benchmark = report.get("benchmark", {}) + reference_synthesis = report.get("reference_synthesis", {}) architecture = architecture_steps(overview) compare_table_rows = compare_rows(compare) benchmark_rows = benchmark_cards(benchmark) + synthesis_rows = synthesis_cards(reference_synthesis) variant_cards = variant_diff_cards(compare) strength_items = "".join(f"
  • {html.escape(item)}
    • " for item in overview.get("strengths", [])) @@ -327,6 +359,23 @@ def render_html(report: dict) -> str: else: benchmark_html = "

    No GitHub benchmark scan has been attached to this package yet.

    " + synthesis_html = "" + if synthesis_rows: + synthesis_html = "".join( + ( + "
    " + f"

    {html.escape(item['name'])}

    " + "

    Borrow now

    " + + ("" if item.get("borrow") else "

    No borrow cues recorded.

    ") + + "

    Avoid

    " + + ("" if item.get("avoid") else "

    No avoid cues recorded.

    ") + + "
    " + ) + for item in synthesis_rows + ) + else: + synthesis_html = "

    No multi-source synthesis has been generated yet.

    " + variant_diff_html = "" if variant_cards: variant_diff_html = "".join( @@ -555,6 +604,7 @@ def render_html(report: dict) -> str: archetype: {html.escape(str(overview.get('metadata', {}).get('skill_archetype', 'scaffold')))} format: {html.escape(str(overview.get('metadata', {}).get('canonical_format', 'agent-skills')))} updated: {html.escape(str(overview.get('metadata', {}).get('updated_at', 'n/a')))} + intent confidence: {html.escape(str(intent_confidence.get('score', 'n/a')))} / 100 @@ -616,6 +666,18 @@ def render_html(report: dict) -> str: +
    +
    +

    Reference synthesis

    +
    {synthesis_html}
    +
    +
    +

    Borrow now

    + +

    {html.escape(reference_synthesis.get('synthesis', {}).get('decision_prompt', 'Run the reference synthesis after the benchmark scan to decide what to borrow next.'))}

    +
    +
    +

    Top three next moves

    {direction_cards}
    diff --git a/scripts/render_skill_overview.py b/scripts/render_skill_overview.py index 5f569b0..40a03d1 100644 --- a/scripts/render_skill_overview.py +++ b/scripts/render_skill_overview.py @@ -70,6 +70,10 @@ def load_benchmark(skill_dir: Path) -> dict: return load_json(skill_dir / "reports" / "github-benchmark-scan.json") +def load_reference_synthesis(skill_dir: Path) -> dict: + return load_json(skill_dir / "reports" / "reference-synthesis.json") + + def extract_title(body: str, fallback: str) -> str: for line in body.splitlines(): if line.startswith("# "): @@ -204,6 +208,10 @@ def benchmark_highlights(benchmark: dict) -> list[dict]: return highlights +def synthesis_highlights(synthesis: dict) -> list[str]: + return synthesis.get("synthesis", {}).get("borrow_now", [])[:3] + + def build_summary(skill_dir: Path) -> dict: skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8") frontmatter, body = parse_frontmatter(skill_text) @@ -221,6 +229,7 @@ def build_summary(skill_dir: Path) -> dict: package_map = package_entries(skill_dir) strengths = derive_strengths(skill_dir, manifest) benchmark = load_benchmark(skill_dir) + reference_synthesis = load_reference_synthesis(skill_dir) return { "name": name, @@ -234,6 +243,7 @@ def build_summary(skill_dir: Path) -> dict: "cards": card_items(interface_data, logic_steps, package_map, usage_steps, description), "introduction": introduction_lines(description), "benchmark_highlights": benchmark_highlights(benchmark), + "synthesis_highlights": synthesis_highlights(reference_synthesis), "metadata": { "canonical_format": interface_data.get("compatibility", {}).get("canonical_format", "agent-skills"), "targets": interface_data.get("compatibility", {}).get("adapter_targets", []), @@ -277,6 +287,7 @@ def render_html(summary: dict) -> str: ) for item in summary.get("benchmark_highlights", []) ) + synthesis_html = "".join(f"
  • {html.escape(item)}
    • " for item in summary.get("synthesis_highlights", [])) return f""" @@ -498,6 +509,16 @@ def render_html(summary: dict) -> str:
    {benchmark_cards or "

    No benchmark scan recorded yet. Run the GitHub benchmark scan first.

    "}
    + +
    +
    +
    +

    Reference synthesis

    +

    These are the strongest cross-source patterns to borrow now after combining GitHub benchmarks with curated official, research, and principle tracks.

    +
    + +
    +
    diff --git a/scripts/yao.py b/scripts/yao.py index 406b6d2..efa44ad 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -6,6 +6,7 @@ import sys from pathlib import Path from github_benchmark_scan import build_query +from render_intent_confidence import assess_intent_confidence ROOT = Path(__file__).resolve().parent.parent @@ -139,6 +140,27 @@ def prompt_optional_entries(label: str) -> list[str]: return [item.strip() for item in value.split(",") if item.strip()] +def update_context_slot(context: dict, slot: str, answer: str, list_mode: bool) -> None: + value = answer.strip() + if not value or value.lower() in {"skip", "none", "no", "n"}: + return + if list_mode: + context[slot] = [item.strip() for item in value.split(",") if item.strip()] + else: + context[slot] = value + + +def intent_confidence_note(summary: dict) -> str: + lines = [ + f"\nIntent confidence: {summary['score']}/100 ({summary['band']}).", + f"- Recommended action: {summary['recommended_action']}", + ] + if summary.get("gaps"): + top_gap = summary["gaps"][0] + lines.append(f"- Biggest gap: {top_gap['label']} — {top_gap['reason']}") + return "\n".join(lines) + "\n" + + ARCHETYPE_MODE = { "scaffold": "scaffold", "production": "production", @@ -215,6 +237,20 @@ def command_init(args: argparse.Namespace) -> int: cmd.extend(["--github-top-n", str(args.github_top_n)]) if args.github_fixture_dir: cmd.extend(["--github-fixture-dir", args.github_fixture_dir]) + if args.intent_job: + cmd.extend(["--intent-job", args.intent_job]) + for item in args.intent_real_input: + cmd.extend(["--intent-real-input", item]) + if args.intent_primary_output: + cmd.extend(["--intent-primary-output", args.intent_primary_output]) + for item in args.intent_exclusion: + cmd.extend(["--intent-exclusion", item]) + for item in args.intent_constraint: + cmd.extend(["--intent-constraint", item]) + for item in args.intent_standard: + cmd.extend(["--intent-standard", item]) + if args.intent_correction: + cmd.extend(["--intent-correction", args.intent_correction]) result = run_script("init_skill.py", cmd) print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) return 0 if result["ok"] else 2 @@ -223,17 +259,31 @@ def command_init(args: argparse.Namespace) -> int: def command_quickstart(args: argparse.Namespace) -> int: sys.stderr.write("Let's start gently. You do not need a polished brief here.\n") sys.stderr.write("Give me the real work in your own words, and I will help turn it into a clean first-pass skill.\n") - sys.stderr.write("Before we deepen the package, I will also look at a few strong public GitHub references so we can borrow patterns without copying them.\n") + sys.stderr.write("Before we deepen the package, I will also look at a few strong public GitHub references, then layer in official, research, and principle tracks so we borrow patterns without copying them.\n") name = args.name or prompt_with_default("Skill name", "my-skill") job = args.job or prompt_with_default( "In your own words, what repeated work do you most want this skill to reliably handle", "Turn a repeated workflow into a reusable skill.", ) + real_inputs = args.real_input or prompt_optional_entries( + "What material will people actually hand to this skill in practice (comma-separated)" + ) primary_output = args.primary_output or prompt_with_default( "If it works beautifully, what should it hand back so you or the next person can keep moving", "A reusable skill package.", ) description = args.description or f"{job.rstrip('.')} Primary output: {primary_output.rstrip('.')}." + intent_context = { + "job": job, + "real_inputs": real_inputs, + "primary_output": primary_output, + "description": description, + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [], + } inferred_archetype, archetype_reason = infer_archetype(job, description) guidance = archetype_guidance(inferred_archetype) sys.stderr.write(discovery_summary(job, primary_output, inferred_archetype, guidance)) @@ -243,10 +293,22 @@ def command_quickstart(args: argparse.Namespace) -> int: ) if correction.lower() not in {"looks right", "skip", "none", "no"}: description = f"{description.rstrip('.')} Keep this correction in scope: {correction.rstrip('.')}." + intent_context["description"] = description + intent_context["correction"] = correction inferred_archetype, archetype_reason = infer_archetype(job, description) guidance = archetype_guidance(inferred_archetype) sys.stderr.write("\nThanks. I tightened the frame before moving on.\n") sys.stderr.write(discovery_summary(job, primary_output, inferred_archetype, guidance)) + confidence = assess_intent_confidence(intent_context) + sys.stderr.write(intent_confidence_note(confidence)) + if not confidence["gate_passed"]: + sys.stderr.write("Before I package this idea, I want to close the highest-leverage gaps instead of guessing.\n") + for follow_up in confidence.get("follow_up_questions", [])[:2]: + answer = prompt_optional(follow_up["question"], "skip") + update_context_slot(intent_context, follow_up["slot"], answer, follow_up["list"]) + confidence = assess_intent_confidence(intent_context) + sys.stderr.write("\nI tightened the intent frame once more before moving on.\n") + sys.stderr.write(intent_confidence_note(confidence)) archetype = args.archetype or prompt_with_default("I would start with this archetype (scaffold/production/library/governed)", inferred_archetype) archetype = archetype if archetype in ARCHETYPE_MODE else inferred_archetype default_mode = ARCHETYPE_MODE[archetype] @@ -260,12 +322,16 @@ def command_quickstart(args: argparse.Namespace) -> int: "If there is anything you admire and want me to learn from as pattern hints, send it here (repo, product, page, workflow; comma-separated)" ) external_references = args.external_reference or [] - local_constraints = args.local_constraint or prompt_optional_entries( + prompted_constraints = args.constraint if getattr(args, "constraint", None) else ([] if args.local_constraint else prompt_optional_entries( "Tell me any local constraints I must keep in view (privacy, naming, compatibility; comma-separated)" - ) + )) + local_constraints = args.local_constraint or prompted_constraints or intent_context.get("constraints", []) + intent_context["user_references"] = user_references + intent_context["constraints"] = local_constraints + confidence = assess_intent_confidence(intent_context) github_query = args.github_query or build_query(" ".join(filter(None, [job, primary_output, description]))) sys.stderr.write(f"GitHub benchmark query: {github_query}\n") - sys.stderr.write("I will use that query to pull three strong public examples, then surface only the patterns worth borrowing or avoiding.\n") + sys.stderr.write("I will use that query to pull three strong public examples, then surface only the patterns worth borrowing or avoiding, plus a compact synthesis from official, research, and principle tracks.\n") title = args.title or name.replace("-", " ").title() guidance = archetype_guidance(archetype) cmd = [ @@ -284,7 +350,21 @@ def command_quickstart(args: argparse.Namespace) -> int: github_query, "--github-top-n", str(args.github_top_n), + "--intent-job", + job, + "--intent-primary-output", + primary_output, ] + for item in real_inputs: + cmd.extend(["--intent-real-input", item]) + for item in intent_context.get("exclusions", []): + cmd.extend(["--intent-exclusion", item]) + for item in intent_context.get("constraints", []): + cmd.extend(["--intent-constraint", item]) + for item in intent_context.get("standards", []): + cmd.extend(["--intent-standard", item]) + if intent_context.get("correction"): + cmd.extend(["--intent-correction", intent_context["correction"]]) if args.github_fixture_dir: cmd.extend(["--github-fixture-dir", args.github_fixture_dir]) for reference in external_references: @@ -318,6 +398,8 @@ def command_quickstart(args: argparse.Namespace) -> int: }, "artifacts": payload.get("artifacts", {}), "github_benchmark_scan": benchmark_scan, + "intent_confidence": confidence, + "reference_synthesis": payload.get("reference_synthesis"), "guidance": { "archetype_reason": archetype_reason, "why_this_mode": ( @@ -329,7 +411,9 @@ def command_quickstart(args: argparse.Namespace) -> int: "focus": guidance["focus"], "next_steps": [ "Open reports/intent-dialogue.md and tighten the real job, outputs, and exclusions.", + "Open reports/intent-confidence.md and close the biggest unresolved intent gap before deepening the package.", "Open reports/github-benchmark-scan.md and review the top three public benchmark repositories plus their borrow and avoid notes.", + "Open reports/reference-synthesis.md and decide which GitHub, official, research, or principle tracks are worth borrowing now.", "Open reports/reference-scan.md and decide which benchmark patterns to borrow and which user references set the quality bar.", "Open reports/review-viewer.html to explain the package to a first-time reviewer.", "Use reports/iteration-directions.md to choose only one high-value next move before adding more files.", @@ -416,12 +500,14 @@ def command_report(args: argparse.Namespace) -> int: run_script("build_confusion_matrix.py", []), run_script("promotion_checker.py", []), run_script("render_eval_dashboard.py", []), + run_script("render_intent_confidence.py", [str(ROOT)]), run_script("render_description_drift_history.py", []), run_script("render_iteration_ledger.py", []), run_script("render_baseline_compare.py", baseline_compare_args()), run_script("render_regression_history.py", []), run_script("render_context_reports.py", []), run_script("render_portability_report.py", []), + run_script("render_reference_synthesis.py", [str(ROOT)]), ] ) report = { @@ -431,11 +517,13 @@ def command_report(args: argparse.Namespace) -> int: "eval_results": "reports/eval_suite.json", "route_scorecard": "reports/route_scorecard.json", "promotion_decisions": "reports/promotion_decisions.json", + "intent_confidence": "reports/intent-confidence.json", "iteration_ledger": "reports/iteration_ledger.md", "baseline_compare": "reports/baseline-compare.json", "regression_history": "reports/regression_history.md", "context_budget": "reports/context_budget.json", "portability_score": "reports/portability_score.json", + "reference_synthesis": "reports/reference-synthesis.json", }, } print(json.dumps(report, ensure_ascii=False, indent=2)) @@ -500,6 +588,20 @@ def command_github_benchmark_scan(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_intent_confidence(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.context_json: + cmd.extend(["--context-json", args.context_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + result = run_script("render_intent_confidence.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_intent_dialogue(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -512,6 +614,18 @@ def command_intent_dialogue(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_reference_synthesis(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + result = run_script("render_reference_synthesis.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_iteration_directions(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -707,6 +821,13 @@ def build_parser() -> argparse.ArgumentParser: init_cmd.add_argument("--github-query") init_cmd.add_argument("--github-top-n", type=int, default=3) init_cmd.add_argument("--github-fixture-dir") + init_cmd.add_argument("--intent-job") + init_cmd.add_argument("--intent-real-input", action="append", default=[]) + init_cmd.add_argument("--intent-primary-output") + init_cmd.add_argument("--intent-exclusion", action="append", default=[]) + init_cmd.add_argument("--intent-constraint", action="append", default=[]) + init_cmd.add_argument("--intent-standard", action="append", default=[]) + init_cmd.add_argument("--intent-correction") init_cmd.set_defaults(func=command_init) quickstart_cmd = subparsers.add_parser( @@ -715,6 +836,7 @@ def build_parser() -> argparse.ArgumentParser: ) quickstart_cmd.add_argument("--name") quickstart_cmd.add_argument("--job") + quickstart_cmd.add_argument("--real-input", action="append", default=[]) quickstart_cmd.add_argument("--primary-output") quickstart_cmd.add_argument("--description") quickstart_cmd.add_argument("--title") @@ -724,6 +846,7 @@ def build_parser() -> argparse.ArgumentParser: quickstart_cmd.add_argument("--external-reference", action="append", default=[]) quickstart_cmd.add_argument("--user-reference", action="append", default=[]) quickstart_cmd.add_argument("--local-constraint", action="append", default=[]) + quickstart_cmd.add_argument("--constraint", action="append", default=[]) quickstart_cmd.add_argument("--github-query") quickstart_cmd.add_argument("--github-top-n", type=int, default=3) quickstart_cmd.add_argument("--github-fixture-dir") @@ -816,6 +939,16 @@ def build_parser() -> argparse.ArgumentParser: github_scan_cmd.add_argument("--output-json") github_scan_cmd.set_defaults(func=command_github_benchmark_scan) + intent_confidence_cmd = subparsers.add_parser( + "intent-confidence", + help="Render a confidence report for how well the real job, inputs, outputs, and exclusions are understood.", + ) + intent_confidence_cmd.add_argument("skill_dir", nargs="?", default=".") + intent_confidence_cmd.add_argument("--context-json") + intent_confidence_cmd.add_argument("--output-md") + intent_confidence_cmd.add_argument("--output-json") + intent_confidence_cmd.set_defaults(func=command_intent_confidence) + intent_dialogue_cmd = subparsers.add_parser( "intent-dialogue", help="Render a front-loaded intent dialogue guide for a skill package.", @@ -825,6 +958,15 @@ def build_parser() -> argparse.ArgumentParser: intent_dialogue_cmd.add_argument("--output-json") intent_dialogue_cmd.set_defaults(func=command_intent_dialogue) + reference_synthesis_cmd = subparsers.add_parser( + "reference-synthesis", + help="Render a multi-source reference synthesis report for a skill package.", + ) + reference_synthesis_cmd.add_argument("skill_dir", nargs="?", default=".") + reference_synthesis_cmd.add_argument("--output-md") + reference_synthesis_cmd.add_argument("--output-json") + reference_synthesis_cmd.set_defaults(func=command_reference_synthesis) + iteration_directions_cmd = subparsers.add_parser( "iteration-directions", help="Render the top three next iteration directions for a skill package.", diff --git a/tests/verify_intent_confidence.py b/tests/verify_intent_confidence.py new file mode 100644 index 0000000..ef3c07d --- /dev/null +++ b/tests/verify_intent_confidence.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_intent_confidence.py" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_intent_confidence" + if tmp_root.exists(): + subprocess.run(["rm", "-rf", str(tmp_root)], check=True) + skill_dir = tmp_root / "intent-confidence-demo" + (skill_dir / "reports").mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text( + "---\nname: intent-confidence-demo\ndescription: Turn repeated incident notes into a reusable incident packet.\n---\n\n# Intent Confidence Demo\n", + encoding="utf-8", + ) + context_path = skill_dir / "reports" / "context.json" + context_path.write_text( + json.dumps( + { + "job": "Turn repeated incident notes into a reusable incident packet.", + "real_inputs": ["incident notes", "chat timeline"], + "primary_output": "A reusable incident command packet.", + "description": "Turn repeated incident notes into a reusable incident packet. Primary output: A reusable incident command packet.", + "exclusions": ["Do not draft external PR statements."], + "constraints": ["auditability", "portability"], + "standards": ["consistency"], + "correction": "", + "user_references": ["A trusted incident workflow"], + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(skill_dir), + "--context-json", + str(context_path), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["summary"]["gate_passed"], payload + assert payload["summary"]["score"] >= 70, payload + markdown = Path(payload["artifacts"]["markdown"]).read_text(encoding="utf-8") + assert "Intent Confidence" in markdown, markdown[:200] + assert "Follow-Up Questions" in markdown, markdown[:500] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_reference_synthesis.py b/tests/verify_reference_synthesis.py new file mode 100644 index 0000000..1a4791e --- /dev/null +++ b/tests/verify_reference_synthesis.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +CLI = ROOT / "scripts" / "yao.py" +BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_reference_synthesis" + if tmp_root.exists(): + subprocess.run(["rm", "-rf", str(tmp_root)], check=True) + + proc = subprocess.run( + [ + sys.executable, + str(CLI), + "init", + "reference-synthesis-demo", + "--description", + "Turn repeated release notes into a portable governed release skill.", + "--output-dir", + str(tmp_root), + "--github-query", + "release workflow evaluation portability", + "--github-fixture-dir", + str(BENCHMARK_FIXTURE_DIR), + "--user-reference", + "A trusted workflow product::taste::Learn the calm operating shape.::Do not copy wording.", + "--intent-job", + "Turn repeated release notes into a portable governed release skill.", + "--intent-real-input", + "release notes", + "--intent-real-input", + "changelog snippets", + "--intent-primary-output", + "A governed release packet.", + "--intent-exclusion", + "Do not publish blog posts.", + "--intent-constraint", + "portability", + "--intent-standard", + "consistency", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + skill_dir = Path(payload["root"]) + synthesis_path = skill_dir / "reports" / "reference-synthesis.json" + assert synthesis_path.exists(), synthesis_path + synthesis = json.loads(synthesis_path.read_text(encoding="utf-8")) + assert len(synthesis["github_benchmarks"]) == 3, synthesis + assert len(synthesis["source_tracks"]) == 3, synthesis + assert synthesis["synthesis"]["borrow_now"], synthesis + markdown = (skill_dir / "reports" / "reference-synthesis.md").read_text(encoding="utf-8") + assert "Curated World-Class Pattern Tracks" in markdown, markdown[:600] + assert "Borrow Now" in markdown, markdown[:900] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_review_viewer.py b/tests/verify_review_viewer.py index 1542e21..a236023 100644 --- a/tests/verify_review_viewer.py +++ b/tests/verify_review_viewer.py @@ -47,6 +47,7 @@ def main() -> None: assert "Compare view" in html_text, html_text[:500] assert "Variant diff studio" in html_text, html_text[:900] assert "Reference coach" in html_text, html_text[:900] + assert "Reference synthesis" in html_text, html_text[:1200] assert "Top three next moves" in html_text, html_text[:500] print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_skill_overview.py b/tests/verify_skill_overview.py index e0b3801..70e4c18 100644 --- a/tests/verify_skill_overview.py +++ b/tests/verify_skill_overview.py @@ -46,8 +46,12 @@ def main() -> None: assert (created / "manifest.json").exists(), created assert (created / "reports" / "intent-dialogue.md").exists(), created assert (created / "reports" / "intent-dialogue.json").exists(), created + assert (created / "reports" / "intent-confidence.md").exists(), created + assert (created / "reports" / "intent-confidence.json").exists(), created assert (created / "reports" / "skill-overview.html").exists(), created assert (created / "reports" / "skill-overview.json").exists(), created + assert (created / "reports" / "reference-synthesis.md").exists(), created + assert (created / "reports" / "reference-synthesis.json").exists(), created assert (created / "reports" / "iteration-directions.md").exists(), created assert (created / "reports" / "iteration-directions.json").exists(), created @@ -61,6 +65,7 @@ def main() -> None: assert "Why It Works" in report_html, report_html[:600] assert "How to introduce this skill" in report_html, report_html[:900] assert "Patterns worth borrowing now" in report_html, report_html[:1200] + assert "Reference synthesis" in report_html, report_html[:1500] intent_text = (created / "reports" / "intent-dialogue.md").read_text(encoding="utf-8") assert "Questions To Ask" in intent_text, intent_text[:400] diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 846c2da..8ff50d9 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -39,9 +39,11 @@ def main() -> None: assert (created / "SKILL.md").exists(), created assert (created / "README.md").exists(), created assert (created / "reports" / "intent-dialogue.md").exists(), created + assert (created / "reports" / "intent-confidence.md").exists(), created assert (created / "reports" / "skill-overview.html").exists(), created assert (created / "reports" / "review-viewer.html").exists(), created assert (created / "reports" / "reference-scan.md").exists(), created + assert (created / "reports" / "reference-synthesis.md").exists(), created assert (created / "reports" / "iteration-directions.md").exists(), created quickstart_result = run( @@ -50,16 +52,32 @@ def main() -> None: str(tmp_root), "--github-fixture-dir", str(BENCHMARK_FIXTURE_DIR), - input_text="quickstart-skill\nTurn rough notes into a reusable package.\nA reusable markdown workflow.\nlooks right\nproduction\nproduction\nA top-tier internal workflow product\nprivacy and naming\n", + input_text=( + "quickstart-skill\n" + "Turn messy release notes into a reusable release brief skill.\n" + "release notes, changelog snippets\n" + "A reusable markdown release brief.\n" + "looks right\n" + "It should not publish blog posts or send email.\n" + "consistency, portability\n" + "production\n" + "production\n" + "A top-tier internal workflow product\n" + "privacy and naming\n" + ), ) assert quickstart_result["ok"], quickstart_result quickstart_root = Path(quickstart_result["payload"]["root"]) assert (quickstart_root / "reports" / "review-viewer.html").exists(), quickstart_root assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root + assert (quickstart_root / "reports" / "intent-confidence.md").exists(), quickstart_root + assert (quickstart_root / "reports" / "reference-synthesis.md").exists(), quickstart_root assert quickstart_result["payload"]["archetype"] == "production", quickstart_result assert len(quickstart_result["payload"]["references"]["benchmark_repositories"]) == 3, quickstart_result assert quickstart_result["payload"]["references"]["user_references"] == ["A top-tier internal workflow product"], quickstart_result assert quickstart_result["payload"]["guidance"]["experience_note"], quickstart_result + assert quickstart_result["payload"]["intent_confidence"]["score"] >= 70, quickstart_result + assert quickstart_result["payload"]["reference_synthesis"]["synthesis"]["borrow_now"], quickstart_result validate_result = run("validate", str(created)) assert validate_result["ok"], validate_result @@ -98,10 +116,18 @@ def main() -> None: assert github_benchmark_result["ok"], github_benchmark_result assert len(github_benchmark_result["payload"]["repositories"]) == 3, github_benchmark_result + intent_confidence_result = run("intent-confidence", str(created)) + assert intent_confidence_result["ok"], intent_confidence_result + assert intent_confidence_result["payload"]["summary"]["score"] >= 0, intent_confidence_result + intent_result = run("intent-dialogue", str(created)) assert intent_result["ok"], intent_result assert intent_result["payload"]["artifacts"]["markdown"].endswith("reports/intent-dialogue.md"), intent_result + reference_synthesis_result = run("reference-synthesis", str(created)) + assert reference_synthesis_result["ok"], reference_synthesis_result + assert reference_synthesis_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-synthesis.md"), reference_synthesis_result + directions_result = run("iteration-directions", str(created)) assert directions_result["ok"], directions_result assert directions_result["payload"]["artifacts"]["markdown"].endswith("reports/iteration-directions.md"), directions_result