From e3ca33c96240855faa27381a818e40f915d61bb3 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Thu, 30 Apr 2026 14:23:44 +0800 Subject: [PATCH] feat: add pattern and output risk guardrails --- .gitignore | 2 + Makefile | 9 +- README.md | 6 + VERSION | 1 + references/authoring-discipline.md | 4 + references/output-quality-risk.md | 40 +++ references/pattern-extraction-doctrine.md | 76 +++++ references/reference-scan.md | 14 + references/skill-engineering-method.md | 36 ++- scripts/check_update.py | 182 ++++++++++++ scripts/ci_test.py | 2 + scripts/init_skill.py | 26 +- scripts/render_output_risk_profile.py | 322 ++++++++++++++++++++++ scripts/render_reference_synthesis.py | 113 ++++++++ scripts/render_review_viewer.py | 107 +++++++ scripts/yao.py | 133 +++++++++ tests/verify_output_risk_profile.py | 60 ++++ tests/verify_reference_synthesis.py | 3 + tests/verify_review_viewer.py | 2 + tests/verify_yao_cli.py | 28 ++ 20 files changed, 1157 insertions(+), 9 deletions(-) create mode 100644 VERSION create mode 100644 references/output-quality-risk.md create mode 100644 references/pattern-extraction-doctrine.md create mode 100644 scripts/check_update.py create mode 100644 scripts/render_output_risk_profile.py create mode 100644 tests/verify_output_risk_profile.py diff --git a/.gitignore b/.gitignore index 6da79bf..de73f3c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ tests/tmp_feedback/ tests/tmp_github_benchmark_scan/ tests/tmp_intent_confidence/ tests/tmp_reference_synthesis/ +tests/tmp_output_risk_profile/ reports/release_snapshots/ +.yao/ diff --git a/Makefile b/Makefile index e4e1699..6a4c8e4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -53,6 +53,9 @@ intent-confidence-check: reference-synthesis-check: $(PYTHON) tests/verify_reference_synthesis.py +output-risk-profile-check: + $(PYTHON) tests/verify_output_risk_profile.py + iteration-directions-check: $(PYTHON) tests/verify_iteration_directions.py @@ -104,10 +107,10 @@ resource-boundary-check: quality-check: $(PYTHON) tests/verify_quality_checks.py -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md diff --git a/README.md b/README.md index 177f0ad..8f56ad3 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ python3 scripts/yao.py reference-scan my-skill \ python3 scripts/yao.py review-viewer my-skill python3 scripts/yao.py feedback my-skill --note "Tighten exclusions before adding scripts." --rating 4 --category boundary python3 scripts/yao.py baseline-compare +python3 scripts/yao.py check-update python3 scripts/yao.py package . --platform generic --output-dir dist ``` @@ -243,6 +244,8 @@ The repository now treats method as a first-class asset instead of scattered gui - [Skill Engineering Method](references/skill-engineering-method.md) - [Intent Dialogue](references/intent-dialogue.md) - [Reference Scan Strategy](references/reference-scan.md) +- [Pattern Extraction Doctrine](references/pattern-extraction-doctrine.md) +- [Output Quality Risk](references/output-quality-risk.md) - [Authoring Discipline](references/authoring-discipline.md) - [Skill Archetypes](references/skill-archetypes.md) - [Gate Selection](references/gate-selection.md) @@ -267,6 +270,7 @@ Most teams keep valuable operating knowledge scattered across chats, personal pr yao-meta-skill/ ├── SKILL.md ├── README.md +├── VERSION ├── LICENSE ├── .gitignore ├── agents/ @@ -315,6 +319,8 @@ Utility scripts that make the meta-skill operational: - `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation - `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage - `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain +- `check_update.py`: checks GitHub for a newer `VERSION` or remote manifest version and reports a reinstall hint without modifying local files +- `render_output_risk_profile.py`: predicts output-specific failure modes such as generic headings, citation clutter, screenshot mistakes, weak Markdown tables, and missing execution assumptions ### `evals/` diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..9084fa2 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.1.0 diff --git a/references/authoring-discipline.md b/references/authoring-discipline.md index ef1847e..02461ab 100644 --- a/references/authoring-discipline.md +++ b/references/authoring-discipline.md @@ -45,7 +45,9 @@ Tie each meaningful change to a check. - trigger changes need route or near-neighbor evidence - execution changes need a sample input, script check, or manual run note +- output-facing changes need an output risk profile and at least one self-repair check - new references need a reason they reduce ambiguity or context cost +- borrowed patterns need recurrence, generativity, distinctiveness, or boundary evidence appropriate to the skill tier - new governance needs an owner, lifecycle expectation, or review cadence - new packaging or portability claims need a concrete target or compatibility check @@ -60,6 +62,8 @@ Before approving a generated or modified skill, check: - the package is no larger than the job requires - changes are limited to the requested scope - each new artifact has a verification reason +- borrowed patterns have reviewer-visible acceptance evidence +- likely output mistakes are named before examples are approved - the next iteration direction is focused, not a bundle of speculative upgrades ## Failure Patterns diff --git a/references/output-quality-risk.md b/references/output-quality-risk.md new file mode 100644 index 0000000..df585da --- /dev/null +++ b/references/output-quality-risk.md @@ -0,0 +1,40 @@ +# Output Quality Risk + +Use this layer when a generated skill produces user-facing artifacts such as tutorials, reports, Markdown pages, screenshots, tables, code snippets, or research summaries. + +## Principle + +A skill is not complete when it can route and execute. It also needs to predict how its final output can fail in small but visible ways, then constrain those failures before the user sees them. + +## Common Failure Modes + +- generic headings that make a tutorial feel templated +- dense footnotes or citation markers that interrupt reading +- tables that render poorly or hide decisions inside long cells +- screenshot references that point to the wrong state, crop, or missing asset +- polished summaries that lose the user's actual audience or scenario +- commands or snippets that omit working directory, inputs, outputs, or side effects + +## Required Author Behavior + +Before finalizing a generated skill: + +1. infer the most likely output families from the job and target output +2. generate `reports/output-risk-profile.md` +3. add output-specific constraints to the generated skill's operating frame +4. expose the risk profile in the review viewer +5. treat unresolved output risks as iteration candidates instead of pretending the first version is complete + +## Self-Repair Rule + +Every output-facing skill should do a final pass for: + +- specificity: headings, titles, and summaries fit the actual domain +- readability: Markdown, tables, and lists remain pleasant to scan +- evidence hygiene: citations support real claims without clutter +- visual truthfulness: screenshots and images are real, relevant, and correctly described +- execution clarity: commands and snippets name their assumptions and expected results + +## Reviewer Rule + +Reviewers should approve the skill only when the likely output mistakes are visible and the generated package contains a reasonable self-repair path for the highest-risk family. diff --git a/references/pattern-extraction-doctrine.md b/references/pattern-extraction-doctrine.md new file mode 100644 index 0000000..499901d --- /dev/null +++ b/references/pattern-extraction-doctrine.md @@ -0,0 +1,76 @@ +# Pattern Extraction Doctrine + +Use this doctrine when a skill borrows ideas from GitHub repositories, products, papers, experts, or user-supplied references. The goal is to extract durable patterns, not copy surface style. + +## Principle + +A borrowed pattern must improve the current skill's reliability, clarity, or portability faster than it increases context cost. + +## Four-Gate Pattern Test + +Accept a pattern only when it passes enough of these gates for the skill's risk tier. + +### 1. Recurrence + +The pattern appears in more than one serious example, source, workflow, or usage scenario. + +Use it to reject one-off tricks that look impressive but have no durable signal. + +### 2. Generativity + +The pattern can guide a new case, not just explain the original example. + +Use it to prefer operating principles, decision rules, and workflow loops over anecdotes. + +### 3. Distinctiveness + +The pattern is more specific than generic good advice. + +Use it to reject empty claims such as "be clear", "be useful", or "add quality" unless the reference shows how. + +### 4. Boundary + +The pattern has a known limit: when not to apply it, what not to borrow, or what cost it introduces. + +Use it to prevent reference scans from becoming unbounded feature expansion. + +## Acceptance Rule + +- `Scaffold`: accept a pattern when it has generativity and boundary clarity. +- `Production`: require recurrence, generativity, and boundary clarity. +- `Library`: require recurrence, generativity, distinctiveness, and boundary clarity. +- `Governed`: require all four gates plus reviewer-visible evidence. + +When the evidence is not strong enough, move the pattern into an iteration candidate instead of the first package. + +## What To Borrow + +Borrow: + +- high-signal workflow loops +- evidence-backed quality gates +- repeatable review checkpoints +- crisp output shapes +- boundary language that prevents route confusion +- portability patterns that preserve meaning across environments + +## What Not To Borrow + +Do not borrow: + +- source branding +- long prose +- roleplay style that does not match the target skill +- heavy research workflows for low-risk scaffolds +- platform-specific assumptions hidden inside a general method +- impressive examples that cannot be verified or generalized + +## Reviewer Questions + +Before accepting a borrowed pattern, ask: + +- Where else does this pattern appear? +- What new case can it help solve? +- What makes it more specific than generic advice? +- When should this skill refuse to use it? +- What file, report, eval, or checklist proves it earned its weight? diff --git a/references/reference-scan.md b/references/reference-scan.md index 1251cf0..0ea9ecb 100644 --- a/references/reference-scan.md +++ b/references/reference-scan.md @@ -65,6 +65,19 @@ Do not turn this into open-ended research. - prefer at least `2` external benchmark objects before treating the scan as complete - if the user provides references, record what they admire and what should explicitly not be copied +## Pattern Acceptance + +Borrowed ideas must pass a lightweight pattern test before they shape the first package. + +- `recurrence`: the pattern appears in more than one serious context or source +- `generativity`: the pattern can guide new cases, not just explain one example +- `distinctiveness`: the pattern is more specific than generic good advice +- `boundary`: the pattern names where not to apply it or what cost it introduces + +For low-risk scaffold work, generativity plus boundary clarity can be enough. For production, library, and governed work, require more evidence before the pattern changes the package shape. + +See [Pattern Extraction Doctrine](pattern-extraction-doctrine.md). + ## Reference Categories Choose the smallest relevant set: @@ -97,6 +110,7 @@ Borrow: - proven gate choices - portable metadata ideas - clear operator-facing flows +- high-signal examples that show the finished experience, not just the internal method Do not borrow: diff --git a/references/skill-engineering-method.md b/references/skill-engineering-method.md index ed57ed8..c674e63 100644 --- a/references/skill-engineering-method.md +++ b/references/skill-engineering-method.md @@ -10,9 +10,10 @@ This doctrine defines the default method for turning messy workflow material int 4. Set one clear capability boundary. 5. Write and test the trigger description before expanding the body. 6. Apply authoring discipline: name unresolved assumptions, keep scope small, and tie meaningful changes to checks. -7. Add only the gates that match the risk. -8. Ship the first routeable package, then pick the three highest-value next iteration directions. -9. Package and govern the skill only as far as real reuse demands. +7. Add an output risk profile for user-facing artifacts. +8. Add only the gates that match the risk. +9. Ship the first routeable package, then pick the three highest-value next iteration directions. +10. Package and govern the skill only as far as real reuse demands. ## Phase 1: Qualification @@ -45,6 +46,17 @@ Before expanding the package, apply the execution discipline that keeps the work See [Authoring Discipline](authoring-discipline.md). +## Phase 1.6: Problem Diagnosis + +When the user gives a fuzzy pain point instead of a clear skill request, diagnose the likely package shape before asking for structure. + +- infer whether the need is best served by a scaffold, production workflow, library capability, governed asset, or no skill +- recommend at most three candidate directions +- explain why each candidate fits, where it is limited, and what the first light version should prove +- prefer a concrete recommendation over a menu when the intent is clear enough + +This keeps discovery useful for users who can describe the problem but not the skill architecture. + ## Phase 2: Intent Dialogue Before deep authoring, ask only the questions that change the package design. @@ -91,10 +103,24 @@ Run a short benchmark pass before deep authoring. - ask for user-supplied references second, but extract only patterns and standards - use local files third, only for fit, privacy, and compatibility calibration - choose from method, structure, execution, portability, and domain patterns -- extract only what improves reliability or clarity +- extract only what passes the pattern test: recurrence, generativity, distinctiveness, and boundary clarity - record what not to borrow so the new skill stays light -See [Reference Scan Strategy](reference-scan.md). +See [Reference Scan Strategy](reference-scan.md) and [Pattern Extraction Doctrine](pattern-extraction-doctrine.md). + +## Phase 5.5: Output Risk Profiling + +Before treating the package as usable, predict the likely mistakes in its final user-facing output. + +- tutorial skills should guard against generic headings, vague examples, and missing success checks +- report and Markdown skills should guard against weak tables, dense lists, and poor hierarchy +- screenshot or visual skills should guard against wrong captures, missing assets, and invented visual evidence +- research or citation-heavy skills should guard against footnote clutter and unsupported claims +- code or command skills should guard against hidden cwd, input, output, and side-effect assumptions + +Generate `reports/output-risk-profile.md` and expose it to the reviewer before adding more structure. + +See [Output Quality Risk](output-quality-risk.md). ## Phase 6: Trigger-First Authoring diff --git a/scripts/check_update.py b/scripts/check_update.py new file mode 100644 index 0000000..050b205 --- /dev/null +++ b/scripts/check_update.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import sys +from datetime import date +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_VERSION_URL = "https://raw.githubusercontent.com/yaojingang/yao-meta-skill/main/VERSION" +DEFAULT_MANIFEST_URL = "https://raw.githubusercontent.com/yaojingang/yao-meta-skill/main/manifest.json" +CACHE_DIR = ROOT / ".yao" +CACHE_PATH = CACHE_DIR / "update-check.json" + + +def load_local_version(root: Path) -> str: + version_file = root / "VERSION" + if version_file.exists(): + value = version_file.read_text(encoding="utf-8").strip() + if value: + return value + manifest_path = root / "manifest.json" + if manifest_path.exists(): + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + version = str(payload.get("version", "")).strip() + if version: + return version + return "0.0.0" + + +def normalize_version(value: str) -> tuple[int, ...]: + parts = re.findall(r"\d+", value) + if not parts: + return (0,) + return tuple(int(part) for part in parts) + + +def is_newer(remote: str, local: str) -> bool: + remote_parts = normalize_version(remote) + local_parts = normalize_version(local) + length = max(len(remote_parts), len(local_parts)) + remote_parts = remote_parts + (0,) * (length - len(remote_parts)) + local_parts = local_parts + (0,) * (length - len(local_parts)) + return remote_parts > local_parts + + +def fetch_text(url: str, timeout: float) -> str: + request = Request(url, headers={"User-Agent": "yao-meta-skill-update-check"}) + with urlopen(request, timeout=timeout) as response: + return response.read().decode("utf-8").strip() + + +def fetch_remote_version(version_url: str, manifest_url: str, timeout: float) -> tuple[str, str]: + try: + value = fetch_text(version_url, timeout) + if value: + return value.splitlines()[0].strip(), version_url + except (HTTPError, URLError, TimeoutError, OSError): + pass + manifest_text = fetch_text(manifest_url, timeout) + payload = json.loads(manifest_text) + version = str(payload.get("version", "")).strip() + if not version: + raise ValueError("Remote manifest does not contain a version.") + return version, manifest_url + + +def read_cache(path: Path) -> dict: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else {} + except (json.JSONDecodeError, OSError): + return {} + + +def write_cache(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def build_result( + local_version: str, + remote_version: str | None, + source: str | None, + checked: bool, + skipped: bool = False, + error: str | None = None, +) -> dict: + update_available = bool(remote_version and is_newer(remote_version, local_version)) + return { + "ok": error is None, + "checked": checked, + "skipped": skipped, + "local_version": local_version, + "remote_version": remote_version, + "update_available": update_available, + "source": source, + "install_hint": "npx skills add yaojingang/yao-meta-skill" if update_available else "", + "error": error, + } + + +def check_update( + root: Path, + cache_path: Path, + version_url: str, + manifest_url: str, + timeout: float, + max_age_days: int, + force: bool, + no_cache: bool, +) -> dict: + local_version = load_local_version(root) + today = str(date.today()) + if not force and not no_cache: + cache = read_cache(cache_path) + if cache.get("checked_at") == today and cache.get("local_version") == local_version: + return { + **build_result( + local_version=local_version, + remote_version=cache.get("remote_version"), + source=cache.get("source"), + checked=False, + skipped=True, + error=cache.get("error"), + ), + "cached": True, + } + try: + remote_version, source = fetch_remote_version(version_url, manifest_url, timeout) + result = build_result(local_version, remote_version, source, checked=True) + except Exception as exc: # noqa: BLE001 - update checks should never break authoring. + result = build_result(local_version, None, None, checked=True, error=str(exc)) + if not no_cache: + write_cache( + cache_path, + { + "checked_at": today, + "local_version": local_version, + "remote_version": result.get("remote_version"), + "source": result.get("source"), + "error": result.get("error"), + }, + ) + result["cached"] = False + result["max_age_days"] = max_age_days + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check whether yao-meta-skill has a newer GitHub version.") + parser.add_argument("--root", default=str(ROOT)) + parser.add_argument("--cache-path", default=str(CACHE_PATH)) + parser.add_argument("--version-url", default=os.environ.get("YAO_UPDATE_VERSION_URL", DEFAULT_VERSION_URL)) + parser.add_argument("--manifest-url", default=os.environ.get("YAO_UPDATE_MANIFEST_URL", DEFAULT_MANIFEST_URL)) + parser.add_argument("--timeout", type=float, default=3.0) + parser.add_argument("--max-age-days", type=int, default=1) + parser.add_argument("--force", action="store_true") + parser.add_argument("--no-cache", action="store_true") + args = parser.parse_args() + result = check_update( + root=Path(args.root).resolve(), + cache_path=Path(args.cache_path).resolve(), + version_url=args.version_url, + manifest_url=args.manifest_url, + timeout=args.timeout, + max_age_days=args.max_age_days, + force=args.force, + no_cache=args.no_cache, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + sys.exit(0 if result["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci_test.py b/scripts/ci_test.py index abba48c..5f9f13a 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -36,6 +36,8 @@ DEFAULT_TARGETS = [ "baseline-compare-check", "reference-scan-check", "github-benchmark-scan-check", + "reference-synthesis-check", + "output-risk-profile-check", "iteration-directions-check", "description-drift-history", "iteration-ledger", diff --git a/scripts/init_skill.py b/scripts/init_skill.py index de282b1..9f1e101 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -8,6 +8,7 @@ from github_benchmark_scan import run_github_benchmark_scan from render_intent_confidence import render_intent_confidence from render_intent_dialogue import render_intent_dialogue from render_iteration_directions import render_iteration_directions +from render_output_risk_profile import render_output_risk_profile from render_reference_scan import parse_reference, render_reference_scan from render_reference_synthesis import render_reference_synthesis from render_review_viewer import render_review_viewer @@ -26,6 +27,18 @@ description: {description} 1. Understand the request. 2. Execute the main task. 3. Validate the result. + +## Output Quality Guardrails + +- Before final output, apply the likely failure modes in `reports/output-risk-profile.md` when that report is present. +- Repair generic headings, cluttered notes, fragile visual assumptions, weak tables, and missing verification cues before handing work back. +- If output-specific evidence is missing, state the gap instead of inventing screenshots, citations, data, or examples. + +## Honest Boundaries + +- Use this skill for the recurring job described in the trigger, not for one-off adjacent requests. +- Treat missing inputs, unclear outputs, or conflicting constraints as reasons to ask one focused clarification. +- Do not add new references, scripts, evals, or governance unless they improve reliability more than they add weight. """ @@ -47,7 +60,14 @@ README_TEMPLATE = """# {title} 6. Follow the workflow steps in `SKILL.md`. 7. Check `reports/skill-overview.html` if you want a fast visual explanation of the package. 8. Open `reports/review-viewer.html` for a compact visual review of the package. -9. Review `reports/iteration-directions.md` for the three most valuable next moves. +9. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. +10. Review `reports/iteration-directions.md` for the three most valuable next moves. + +## Honest Boundaries + +- This package starts from the current intent frame and should not pretend to cover unclear adjacent jobs. +- The first version should ask for clarification when the real input, output, or exclusion boundary is still fuzzy. +- New structure should be added only when it earns its keep through evidence, validation, or reviewer need. ## Package Map @@ -59,6 +79,7 @@ README_TEMPLATE = """# {title} - `reports/github-benchmark-scan.md`: top public benchmark repositories, extracted patterns, and borrow or avoid notes - `reports/reference-scan.md`: benchmark notes from public references, user references, and local constraints - `reports/reference-synthesis.md`: a combined view of GitHub benchmarks plus curated world-class pattern tracks +- `reports/output-risk-profile.md`: predicted output failure modes and self-repair constraints for this skill - `reports/skill-overview.html`: visual overview report - `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps - `reports/iteration-directions.md`: the top three next iteration directions @@ -224,6 +245,7 @@ def initialize_skill( dedupe_references([*combined_external_references, *(user_references or []), *(local_constraints or [])]), ) reference_synthesis = render_reference_synthesis(root) + output_risk_profile = render_output_risk_profile(root) overview = render_skill_overview(root) iteration_directions = render_iteration_directions(root) review_viewer = render_review_viewer(root) @@ -241,6 +263,8 @@ def initialize_skill( "reference_scan_json": reference_scan["artifacts"]["json"], "reference_synthesis_md": reference_synthesis["artifacts"]["markdown"], "reference_synthesis_json": reference_synthesis["artifacts"]["json"], + "output_risk_profile_md": output_risk_profile["artifacts"]["markdown"], + "output_risk_profile_json": output_risk_profile["artifacts"]["json"], "iteration_directions_md": iteration_directions["artifacts"]["markdown"], "iteration_directions_json": iteration_directions["artifacts"]["json"], "review_viewer_html": review_viewer["artifacts"]["html"], diff --git a/scripts/render_output_risk_profile.py b/scripts/render_output_risk_profile.py new file mode 100644 index 0000000..3c696cd --- /dev/null +++ b/scripts/render_output_risk_profile.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +RISK_FAMILIES = [ + { + "key": "tutorial_quality", + "label": "Tutorial quality", + "keywords": ["tutorial", "guide", "how to", "lesson", "course", "walkthrough", "教程", "指南", "课程"], + "risks": [ + "Generic section headings make the tutorial feel templated instead of fitted to the learner's task.", + "Steps may explain what to do without naming the exact check that proves the step worked.", + "Examples can become too abstract when the input material is concrete.", + ], + "constraints": [ + "Write headings from the user's domain nouns and desired outcome, not from generic labels like Overview or Key Points.", + "Pair each major step with a visible success check or expected intermediate output.", + "Use one concrete worked example before adding broad principles.", + ], + "self_repair": [ + "Replace generic H2/H3 headings with task-specific headings before final output.", + "Scan every numbered step for a missing verification cue.", + ], + }, + { + "key": "markdown_readability", + "label": "Markdown readability", + "keywords": ["markdown", "md", "table", "report", "brief", "doc", "文档", "表格", "报告", "排版"], + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability.", + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome.", + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards.", + ], + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "keywords": ["citation", "source", "research", "footnote", "reference", "引用", "角标", "来源", "注释"], + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself.", + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote.", + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note.", + ], + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "keywords": ["screenshot", "image", "visual", "screen", "capture", "figma", "截图", "图片", "视觉", "截屏"], + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region.", + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists.", + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state.", + ], + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "keywords": ["code", "script", "cli", "command", "terminal", "api", "代码", "脚本", "命令", "接口"], + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered.", + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code.", + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode.", + ], + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "keywords": ["marketing", "copy", "title", "article", "content", "summary", "标题", "文案", "文章", "总结"], + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints.", + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input.", + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged.", + ], + }, +] + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {}, text + frontmatter_text = "\n".join(lines[1:end_index]) + body = "\n".join(lines[end_index + 1 :]).lstrip() + if yaml is not None: + payload = yaml.safe_load(frontmatter_text) or {} + return payload if isinstance(payload, dict) else {}, body + data = {} + for line in frontmatter_text.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data, body + + +def load_json(path: Path) -> dict: + if not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def normalized_text(skill_dir: Path) -> str: + skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8") + frontmatter, body = parse_frontmatter(skill_text) + intent = load_json(skill_dir / "reports" / "intent-confidence.json") + context = intent.get("context", {}) if isinstance(intent, dict) else {} + parts = [ + skill_dir.name, + str(frontmatter.get("name", "")), + str(frontmatter.get("description", "")), + body, + str(context.get("job", "")), + str(context.get("primary_output", "")), + " ".join(context.get("real_inputs", []) or []), + " ".join(context.get("constraints", []) or []), + " ".join(context.get("standards", []) or []), + ] + return " ".join(parts).lower() + + +def match_family(text: str, family: dict) -> tuple[int, list[str]]: + hits = [] + for keyword in family["keywords"]: + if keyword.lower() in text: + hits.append(keyword) + return len(hits), hits + + +def build_summary(skill_dir: Path) -> dict: + text = normalized_text(skill_dir) + matched = [] + for family in RISK_FAMILIES: + score, hits = match_family(text, family) + if score: + matched.append({**family, "score": score, "matched_keywords": hits}) + if not matched: + matched = [ + { + **next(item for item in RISK_FAMILIES if item["key"] == "tone_and_specificity"), + "score": 0, + "matched_keywords": ["general-output-risk"], + }, + { + **next(item for item in RISK_FAMILIES if item["key"] == "markdown_readability"), + "score": 0, + "matched_keywords": ["general-output-risk"], + }, + ] + matched = sorted(matched, key=lambda item: item["score"], reverse=True)[:4] + top_risks = [] + constraints = [] + self_repair = [] + for item in matched: + top_risks.extend(item["risks"][:2]) + constraints.extend(item["constraints"][:2]) + self_repair.extend(item["self_repair"][:2]) + return { + "skill_name": skill_dir.name, + "risk_families": [ + { + "key": item["key"], + "label": item["label"], + "matched_keywords": item["matched_keywords"], + "score": item["score"], + "risks": item["risks"], + "constraints": item["constraints"], + "self_repair": item["self_repair"], + } + for item in matched + ], + "top_risks": dedupe(top_risks, 6), + "output_constraints": dedupe(constraints, 6), + "self_repair_checks": dedupe(self_repair, 6), + "reviewer_note": "Use this report before deepening the package and again before approving example outputs.", + } + + +def dedupe(items: list[str], limit: int) -> list[str]: + seen = set() + output = [] + for item in items: + if item in seen: + continue + seen.add(item) + output.append(item) + if len(output) == limit: + break + return output + + +def render_markdown(summary: dict) -> str: + lines = [ + "# Output Risk Profile", + "", + f"Skill: `{summary['skill_name']}`", + "", + "## Why This Exists", + "", + "Generated skills often fail in small output details: generic headings, cluttered citations, fragile screenshots, weak Markdown rendering, or missing execution assumptions. This profile predicts the most likely output mistakes before the skill is used heavily.", + "", + "## Matched Risk Families", + "", + ] + for family in summary["risk_families"]: + lines.extend( + [ + f"### {family['label']}", + f"- Matched keywords: {', '.join(family['matched_keywords'])}", + f"- Score: `{family['score']}`", + "", + ] + ) + lines.extend(["## Likely Output Mistakes", ""]) + for item in summary["top_risks"]: + lines.append(f"- {item}") + lines.extend(["", "## Output Constraints To Apply", ""]) + for item in summary["output_constraints"]: + lines.append(f"- {item}") + lines.extend(["", "## Self-Repair Checks", ""]) + for item in summary["self_repair_checks"]: + lines.append(f"- {item}") + lines.extend(["", "## Reviewer Note", "", summary["reviewer_note"], ""]) + return "\n".join(lines).strip() + "\n" + + +def render_output_risk_profile( + skill_dir: Path, + output_md: Path | None = None, + output_json: Path | None = None, +) -> dict: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + output_md = output_md or reports_dir / "output-risk-profile.md" + output_json = output_json or reports_dir / "output-risk-profile.json" + summary = build_summary(skill_dir) + output_md.write_text(render_markdown(summary), encoding="utf-8") + output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return { + "ok": True, + "skill_dir": str(skill_dir), + "artifacts": { + "markdown": str(output_md), + "json": str(output_json), + }, + "summary": summary, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render predicted output failure modes and quality constraints.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-md") + parser.add_argument("--output-json") + args = parser.parse_args() + result = render_output_risk_profile( + Path(args.skill_dir), + output_md=Path(args.output_md).resolve() if args.output_md else None, + output_json=Path(args.output_json).resolve() if args.output_json else None, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_reference_synthesis.py b/scripts/render_reference_synthesis.py index 574c151..67cb11a 100644 --- a/scripts/render_reference_synthesis.py +++ b/scripts/render_reference_synthesis.py @@ -156,6 +156,99 @@ def select_source_tracks(text: str) -> list[dict[str, Any]]: return selected +def pattern_gate_threshold(manifest: dict[str, Any]) -> int: + tier = str(manifest.get("maturity_tier") or manifest.get("skill_archetype") or "scaffold").lower() + if tier in {"governed", "library"}: + return 4 + if tier == "production": + return 3 + return 2 + + +def score_pattern(candidate: dict[str, Any], source_count: int) -> dict[str, Any]: + text = " ".join( + str(candidate.get(key, "")) + for key in ("name", "borrow", "avoid", "why_relevant", "source_type", "evidence_mode") + ).lower() + gates = { + "recurrence": source_count > 1 + or any(token in text for token in ("repeat", "loop", "workflow", "repositories", "benchmark", "cross")), + "generativity": any( + token in text + for token in ("guide", "loop", "workflow", "pattern", "principle", "operator", "boundary", "output") + ), + "distinctiveness": not any( + phrase in text + for phrase in ("be clear", "be useful", "good quality", "general fit") + ), + "boundary": bool(candidate.get("avoid")) or any(token in text for token in ("avoid", "not", "boundary", "cost")), + } + passed = [name for name, ok in gates.items() if ok] + missing = [name for name, ok in gates.items() if not ok] + return { + "name": candidate.get("name", "Unknown pattern"), + "source_type": candidate.get("source_type", "unknown"), + "borrow": candidate.get("borrow", ""), + "avoid": candidate.get("avoid", ""), + "gates": gates, + "passed": passed, + "missing": missing, + "score": len(passed), + } + + +def build_pattern_gate( + source_tracks: list[dict[str, Any]], + benchmark: dict[str, Any], + user_refs: list[dict[str, Any]], + manifest: dict[str, Any], +) -> dict[str, Any]: + threshold = pattern_gate_threshold(manifest) + source_count = len(source_tracks) + len(benchmark.get("repositories", [])) + len(user_refs) + candidates = [] + for track in source_tracks: + candidates.append(score_pattern(track, source_count)) + for repo in benchmark.get("repositories", [])[:3]: + candidates.append( + score_pattern( + { + "name": repo.get("full_name", "GitHub benchmark"), + "source_type": "github", + "borrow": "; ".join(repo.get("borrow", [])[:2]), + "avoid": "; ".join(repo.get("avoid", [])[:1]), + "why_relevant": "Top GitHub benchmark object with concrete package cues.", + "evidence_mode": "github-benchmark", + }, + source_count, + ) + ) + for reference in user_refs[:3]: + candidates.append( + score_pattern( + { + "name": reference.get("name", "User reference"), + "source_type": "user-reference", + "borrow": reference.get("borrow", ""), + "avoid": reference.get("avoid", ""), + "why_relevant": "User-supplied taste or quality reference.", + "evidence_mode": "user-reference", + }, + source_count, + ) + ) + accepted = [candidate for candidate in candidates if candidate["score"] >= threshold] + deferred = [candidate for candidate in candidates if candidate["score"] < threshold] + return { + "threshold": threshold, + "source_count": source_count, + "accepted": accepted, + "deferred": deferred, + "summary": ( + f"{len(accepted)} accepted, {len(deferred)} deferred using threshold {threshold}/4." + ), + } + + def unique_items(items: list[str], limit: int) -> list[str]: seen = set() output = [] @@ -345,6 +438,7 @@ def build_summary(skill_dir: Path) -> dict[str, Any]: track_borrow = [track["borrow"] for track in source_tracks] track_avoid = [track["avoid"] for track in source_tracks] user_refs = reference_scan.get("user_references", []) + pattern_gate = build_pattern_gate(source_tracks, benchmark, user_refs, manifest) borrow_now = unique_items( [ @@ -397,6 +491,7 @@ def build_summary(skill_dir: Path) -> dict[str, Any]: "borrow_now": borrow_now, "avoid_now": avoid_now, "quality_risers": quality_risers, + "pattern_gate": pattern_gate, "conflicts": conflicts, "recommendation": recommendation, "visibility": visibility, @@ -461,6 +556,24 @@ def render_markdown(summary: dict[str, Any]) -> str: for item in summary["synthesis"]["avoid_now"]: lines.append(f"- {item}") + lines.extend(["", "## Pattern Gate", ""]) + pattern_gate = summary["synthesis"]["pattern_gate"] + lines.append(f"- Summary: {pattern_gate['summary']}") + lines.append(f"- Acceptance threshold: `{pattern_gate['threshold']}/4`") + if pattern_gate["accepted"]: + lines.append("- Accepted patterns:") + for item in pattern_gate["accepted"][:5]: + lines.append( + f" - **{item['name']}**: {item['score']}/4 " + f"({', '.join(item['passed'])})" + ) + if pattern_gate["deferred"]: + lines.append("- Deferred patterns:") + for item in pattern_gate["deferred"][:5]: + lines.append( + f" - **{item['name']}**: missing {', '.join(item['missing']) or 'none'}" + ) + lines.extend(["", "## Default Recommendation", ""]) lines.append(f"- Summary: {summary['synthesis']['recommendation']['summary']}") lines.append(f"- Why: {summary['synthesis']['recommendation']['why']}") diff --git a/scripts/render_review_viewer.py b/scripts/render_review_viewer.py index a60a24e..32f74e0 100644 --- a/scripts/render_review_viewer.py +++ b/scripts/render_review_viewer.py @@ -8,6 +8,7 @@ from pathlib import Path from render_intent_confidence import render_intent_confidence from render_intent_dialogue import render_intent_dialogue from render_iteration_directions import render_iteration_directions +from render_output_risk_profile import render_output_risk_profile from render_reference_scan import render_reference_scan from render_reference_synthesis import render_reference_synthesis from render_skill_overview import render_skill_overview @@ -56,12 +57,18 @@ def load_reference_synthesis_summary(skill_dir: Path) -> dict: return payload if isinstance(payload, dict) else {} +def load_output_risk_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "output-risk-profile.json") + return payload if isinstance(payload, dict) else {} + + def ensure_report_inputs(skill_dir: Path) -> dict: overview_json = skill_dir / "reports" / "skill-overview.json" intent_confidence_json = skill_dir / "reports" / "intent-confidence.json" intent_json = skill_dir / "reports" / "intent-dialogue.json" reference_json = skill_dir / "reports" / "reference-scan.json" reference_synthesis_json = skill_dir / "reports" / "reference-synthesis.json" + output_risk_json = skill_dir / "reports" / "output-risk-profile.json" directions_json = skill_dir / "reports" / "iteration-directions.json" overview_payload = load_json(overview_json) if overview_json.exists() else {} @@ -69,12 +76,14 @@ def ensure_report_inputs(skill_dir: Path) -> dict: intent_payload = load_json(intent_json) if intent_json.exists() else {} reference_payload = load_json(reference_json) if reference_json.exists() else {} reference_synthesis_payload = load_json(reference_synthesis_json) if reference_synthesis_json.exists() else {} + output_risk_payload = load_json(output_risk_json) if output_risk_json.exists() else {} directions_payload = load_json(directions_json) if directions_json.exists() else {} intent_confidence = intent_confidence_payload or render_intent_confidence(skill_dir)["summary"] intent = intent_payload or render_intent_dialogue(skill_dir)["summary"] reference = reference_payload or render_reference_scan(skill_dir, [])["summary"] reference_synthesis = reference_synthesis_payload or render_reference_synthesis(skill_dir)["summary"] + output_risk = output_risk_payload or render_output_risk_profile(skill_dir)["summary"] overview = overview_payload or render_skill_overview(skill_dir)["summary"] iteration = directions_payload.get("summary", {}) or render_iteration_directions(skill_dir)["summary"] feedback = load_feedback_summary(skill_dir) @@ -83,6 +92,7 @@ def ensure_report_inputs(skill_dir: Path) -> dict: promotion = load_specific_promotion(skill_dir) benchmark = load_benchmark_summary(skill_dir) reference_synthesis = load_reference_synthesis_summary(skill_dir) + output_risk = load_output_risk_summary(skill_dir) or output_risk return { "overview": overview, "intent_confidence": intent_confidence, @@ -95,6 +105,7 @@ def ensure_report_inputs(skill_dir: Path) -> dict: "promotion": promotion, "benchmark": benchmark, "reference_synthesis": reference_synthesis, + "output_risk": output_risk, } @@ -223,6 +234,50 @@ def variant_diff_cards(compare: dict) -> list[dict]: return cards +def evidence_readiness(report: dict) -> dict: + intent_confidence = report.get("intent_confidence", {}) + reference_synthesis = report.get("reference_synthesis", {}) + output_risk = report.get("output_risk", {}) + benchmark = report.get("benchmark", {}) + synthesis = reference_synthesis.get("synthesis", {}) if isinstance(reference_synthesis, dict) else {} + pattern_gate = synthesis.get("pattern_gate", {}) if isinstance(synthesis, dict) else {} + accepted_patterns = pattern_gate.get("accepted", []) if isinstance(pattern_gate, dict) else [] + conflicts = synthesis.get("conflicts", []) if isinstance(synthesis, dict) else [] + checks = [ + { + "label": "Intent clarity", + "status": "ready" if intent_confidence.get("gate_passed") else "needs review", + "detail": f"{intent_confidence.get('score', 0)}/100 intent confidence.", + }, + { + "label": "Benchmark coverage", + "status": "ready" if len(benchmark.get("repositories", [])) >= 2 else "needs evidence", + "detail": f"{len(benchmark.get('repositories', []))} GitHub benchmark repositories attached.", + }, + { + "label": "Pattern gate", + "status": "ready" if accepted_patterns else "needs review", + "detail": pattern_gate.get("summary", "No pattern gate summary attached."), + }, + { + "label": "Conflict handling", + "status": "ready" if not conflicts else "decision needed", + "detail": "No material conflicts detected." if not conflicts else conflicts[0].get("summary", "Conflict detected."), + }, + { + "label": "Output risk profile", + "status": "ready" if output_risk.get("risk_families") else "needs review", + "detail": f"{len(output_risk.get('risk_families', []))} output risk families attached.", + }, + ] + ready_count = sum(1 for item in checks if item["status"] == "ready") + return { + "score": int(ready_count / len(checks) * 100), + "checks": checks, + "reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery.", + } + + def render_html(report: dict) -> str: overview = report["overview"] intent = report["intent"] @@ -236,11 +291,13 @@ def render_html(report: dict) -> str: promotion = report.get("promotion", {}) benchmark = report.get("benchmark", {}) reference_synthesis = report.get("reference_synthesis", {}) + output_risk = report.get("output_risk", {}) architecture = architecture_steps(overview) compare_table_rows = compare_rows(compare) benchmark_rows = benchmark_cards(benchmark) synthesis_rows = synthesis_cards(reference_synthesis) variant_cards = variant_diff_cards(compare) + readiness = evidence_readiness(report) strength_items = "".join(f"
  • {html.escape(item)}
    • " for item in overview.get("strengths", [])) logic_items = "".join(f"
  • {html.escape(item)}
    • " for item in overview.get("logic_steps", [])) @@ -260,6 +317,28 @@ def render_html(report: dict) -> str: if not reference_items: reference_items = "
  • No external benchmark objects recorded yet. Add 2 to 5 references before deepening the package.
    • " + output_risk_items = "".join( + ( + "
  • " + f"{html.escape(item.get('label', item.get('key', 'Risk')))}
    " + f"{html.escape('; '.join(item.get('risks', [])[:2]))}" + "
    • " + ) + for item in output_risk.get("risk_families", [])[:3] + ) + if not output_risk_items: + output_risk_items = "
  • No output risk profile attached yet. Generate one before approving example outputs.
    • " + + readiness_html = "".join( + ( + "
  • " + f"{html.escape(item['label'])} · {html.escape(item['status'])}
    " + f"{html.escape(item['detail'])}" + "
    • " + ) + for item in readiness["checks"] + ) + direction_cards = "".join( ( "
    " @@ -651,6 +730,33 @@ def render_html(report: dict) -> str:
    {variant_diff_html}
    +
    +
    +

    Evidence readiness

    +

    Readiness score: {html.escape(str(readiness['score']))}/100

    +
      {readiness_html}
    +
    +
    +

    Honest boundary check

    +
      +
    • Are the known limits visible before the package deepens?
      • +
    • Does the evidence support the borrowed patterns?
      • +
    • Should uncertainty become a clarification question instead of more structure?
      • +
    +
    +
    + +
    +
    +

    Output risk profile

    +
      {output_risk_items}
    +
    +
    +

    Self-repair checks

    +
      {"".join(f"
    • {html.escape(item)}
      • " for item in output_risk.get('self_repair_checks', [])[:5]) or "
    • No self-repair checks attached yet.
      • "}
    +
    +
    +

    Reference coach

    @@ -744,6 +850,7 @@ def render_review_viewer(skill_dir: Path, output_html: Path | None = None, outpu output_json = output_json or reports_dir / "review-viewer.json" report = ensure_report_inputs(skill_dir) + report["evidence_readiness"] = evidence_readiness(report) output_html.write_text(render_html(report), encoding="utf-8") output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") return { diff --git a/scripts/yao.py b/scripts/yao.py index 73af2b6..f30f92b 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -213,6 +213,69 @@ def discovery_summary(job: str, primary_output: str, archetype: str, guidance: d ) +def explicit_skill_request(job: str, description: str) -> bool: + text = f"{job} {description}".lower() + return any(token in text for token in ("skill", "workflow", "checklist", "package", "automate", "standardize")) + + +def diagnose_skill_candidates(job: str, primary_output: str, archetype: str, confidence: dict) -> dict: + fuzzy = not explicit_skill_request(job, primary_output) or confidence.get("score", 0) < 75 + candidates = [ + { + "shape": archetype, + "recommendation": "recommended", + "why_it_fits": "This is the lightest shape that matches the current recurring job signal.", + "limitation": "It should not deepen until the concrete output and exclusion boundary are clear.", + "first_pass": "Create one routeable skill with honest boundaries, one review report, and one next-step direction.", + } + ] + if archetype != "scaffold": + candidates.append( + { + "shape": "scaffold", + "recommendation": "fallback", + "why_it_fits": "Use this if the idea is still exploratory or personal.", + "limitation": "It may under-serve team reuse, portability, or governance needs.", + "first_pass": "Ship only SKILL.md, interface metadata, intent confidence, and review viewer.", + } + ) + if archetype not in {"production", "governed"}: + candidates.append( + { + "shape": "production", + "recommendation": "upgrade path", + "why_it_fits": "Use this when the workflow will be repeated by a team or needs consistent outputs.", + "limitation": "It adds validation and review cost that a personal scaffold may not need.", + "first_pass": "Add one practical eval or execution check after the trigger boundary is stable.", + } + ) + if archetype != "governed" and any(token in f"{job} {primary_output}".lower() for token in ("risk", "audit", "release", "policy", "security", "compliance")): + candidates.append( + { + "shape": "governed", + "recommendation": "risk path", + "why_it_fits": "Use this if the skill affects operational, compliance, security, or release decisions.", + "limitation": "It is too heavy unless ownership and review cadence are real.", + "first_pass": "Add owner, review cadence, lifecycle metadata, and reviewer-visible evidence.", + } + ) + return { + "mode": "fuzzy-problem-diagnosis" if fuzzy else "direct-skill-shaping", + "fuzzy": fuzzy, + "candidates": candidates[:3], + } + + +def diagnosis_note(diagnosis: dict) -> str: + lines = ["\nProblem-to-skill diagnosis:"] + for candidate in diagnosis["candidates"]: + lines.append( + f"- {candidate['shape']} ({candidate['recommendation']}): {candidate['why_it_fits']} " + f"First pass: {candidate['first_pass']}" + ) + return "\n".join(lines) + "\n" + + def reference_visibility(reference_synthesis: dict) -> dict: synthesis = reference_synthesis.get("synthesis", {}) if isinstance(reference_synthesis, dict) else {} visibility = synthesis.get("visibility", {}) if isinstance(synthesis, dict) else {} @@ -246,6 +309,21 @@ def recommendation_from_synthesis(reference_synthesis: dict, visibility: dict) - } +def maybe_emit_update_notice(args: argparse.Namespace) -> None: + if getattr(args, "no_update_check", False): + return + result = run_script("check_update.py", []) + payload = result["payload"] if result["payload"] is not None else {} + if not result["ok"] and not payload: + return + if payload.get("update_available"): + sys.stderr.write( + "\nUpdate available for yao-meta-skill: " + f"{payload.get('local_version')} -> {payload.get('remote_version')}.\n" + f"Run: {payload.get('install_hint')}\n" + ) + + def command_init(args: argparse.Namespace) -> int: cmd = [ args.name, @@ -290,6 +368,7 @@ def command_init(args: argparse.Namespace) -> int: def command_quickstart(args: argparse.Namespace) -> int: + maybe_emit_update_notice(args) sys.stderr.write("Let's start gently. You do not need a polished brief here.\n") sys.stderr.write("Give me the real work in your own words, and I will help turn it into a clean first-pass skill.\n") sys.stderr.write("While we shape the first pass, I will quietly check a few strong public patterns in the background and only surface them if there is real uncertainty or a design conflict.\n") @@ -334,6 +413,9 @@ def command_quickstart(args: argparse.Namespace) -> int: sys.stderr.write(discovery_summary(job, primary_output, inferred_archetype, guidance)) confidence = assess_intent_confidence(intent_context) sys.stderr.write(intent_confidence_note(confidence)) + diagnosis = diagnose_skill_candidates(job, primary_output, inferred_archetype, confidence) + if diagnosis["fuzzy"]: + sys.stderr.write(diagnosis_note(diagnosis)) if not confidence["gate_passed"]: sys.stderr.write("Before I package this idea, I want to close the highest-leverage gaps instead of guessing.\n") for follow_up in confidence.get("follow_up_questions", [])[:2]: @@ -342,11 +424,15 @@ def command_quickstart(args: argparse.Namespace) -> int: confidence = assess_intent_confidence(intent_context) sys.stderr.write("\nI tightened the intent frame once more before moving on.\n") sys.stderr.write(intent_confidence_note(confidence)) + diagnosis = diagnose_skill_candidates(job, primary_output, inferred_archetype, confidence) + if diagnosis["fuzzy"]: + sys.stderr.write(diagnosis_note(diagnosis)) archetype = args.archetype or prompt_with_default("I would start with this archetype (scaffold/production/library/governed)", inferred_archetype) archetype = archetype if archetype in ARCHETYPE_MODE else inferred_archetype default_mode = ARCHETYPE_MODE[archetype] mode = args.mode or prompt_with_default("For the first pass, I would keep the mode here (scaffold/production/library/governed)", default_mode) mode = mode if mode in ARCHETYPE_MODE.values() else default_mode + diagnosis = diagnose_skill_candidates(job, primary_output, archetype, confidence) guidance = archetype_guidance(archetype) sys.stderr.write( f"\nGood. I will treat this as `{archetype}` in `{mode}` mode, so the first pass stays focused on {guidance['focus']}.\n" @@ -455,6 +541,7 @@ def command_quickstart(args: argparse.Namespace) -> int: }, "guidance": { "archetype_reason": archetype_reason, + "problem_diagnosis": diagnosis, "why_this_mode": ( "Scaffold mode keeps the first package light and lets you postpone governance-heavy work until reuse becomes real." if mode == "scaffold" @@ -679,6 +766,17 @@ def command_reference_synthesis(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_output_risk_profile(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + result = run_script("render_output_risk_profile.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_iteration_directions(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -857,6 +955,23 @@ def command_test(args: argparse.Namespace) -> int: return 0 if report["ok"] else 2 +def command_check_update(args: argparse.Namespace) -> int: + cmd = [] + if args.force: + cmd.append("--force") + if args.no_cache: + cmd.append("--no-cache") + if args.version_url: + cmd.extend(["--version-url", args.version_url]) + if args.manifest_url: + cmd.extend(["--manifest-url", args.manifest_url]) + if args.timeout is not None: + cmd.extend(["--timeout", str(args.timeout)]) + result = run_script("check_update.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Unified authoring CLI for yao-meta-skill.") subparsers = parser.add_subparsers(dest="command", required=True) @@ -903,6 +1018,7 @@ def build_parser() -> argparse.ArgumentParser: quickstart_cmd.add_argument("--github-query") quickstart_cmd.add_argument("--github-top-n", type=int, default=3) quickstart_cmd.add_argument("--github-fixture-dir") + quickstart_cmd.add_argument("--no-update-check", action="store_true") quickstart_cmd.set_defaults(func=command_quickstart) validate_cmd = subparsers.add_parser("validate", help="Run validate, lint, governance, and resource checks.") @@ -1020,6 +1136,15 @@ def build_parser() -> argparse.ArgumentParser: reference_synthesis_cmd.add_argument("--output-json") reference_synthesis_cmd.set_defaults(func=command_reference_synthesis) + output_risk_cmd = subparsers.add_parser( + "output-risk-profile", + help="Render predicted output failure modes and self-repair checks for a skill package.", + ) + output_risk_cmd.add_argument("skill_dir", nargs="?", default=".") + output_risk_cmd.add_argument("--output-md") + output_risk_cmd.add_argument("--output-json") + output_risk_cmd.set_defaults(func=command_output_risk_profile) + iteration_directions_cmd = subparsers.add_parser( "iteration-directions", help="Render the top three next iteration directions for a skill package.", @@ -1058,6 +1183,14 @@ def build_parser() -> argparse.ArgumentParser: test_cmd.add_argument("--target", default="test") test_cmd.set_defaults(func=command_test) + update_cmd = subparsers.add_parser("check-update", help="Check whether a newer yao-meta-skill version is available.") + update_cmd.add_argument("--force", action="store_true") + update_cmd.add_argument("--no-cache", action="store_true") + update_cmd.add_argument("--version-url") + update_cmd.add_argument("--manifest-url") + update_cmd.add_argument("--timeout", type=float, default=3.0) + update_cmd.set_defaults(func=command_check_update) + return parser diff --git a/tests/verify_output_risk_profile.py b/tests/verify_output_risk_profile.py new file mode 100644 index 0000000..6f1ec2f --- /dev/null +++ b/tests/verify_output_risk_profile.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_output_risk_profile" + if tmp_root.exists(): + subprocess.run(["rm", "-rf", str(tmp_root)], check=True) + tmp_root.mkdir(parents=True, exist_ok=True) + + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init_skill.py"), + "tutorial-risk-demo", + "--description", + "Create Markdown tutorials with screenshots, tables, citations, and personalized section headings.", + "--output-dir", + str(tmp_root), + "--intent-primary-output", + "A polished Markdown tutorial.", + "--intent-real-input", + "screenshots", + "--intent-real-input", + "source notes", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + skill_dir = tmp_root / "tutorial-risk-demo" + proc = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "render_output_risk_profile.py"), str(skill_dir)], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(proc.stdout) + summary = payload["summary"] + keys = {item["key"] for item in summary["risk_families"]} + assert "tutorial_quality" in keys, summary + assert "markdown_readability" in keys, summary + assert summary["output_constraints"], summary + assert summary["self_repair_checks"], summary + markdown = (skill_dir / "reports" / "output-risk-profile.md").read_text(encoding="utf-8") + assert "Likely Output Mistakes" in markdown, markdown[:800] + assert "Self-Repair Checks" in markdown, markdown[:1200] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_reference_synthesis.py b/tests/verify_reference_synthesis.py index 3ec3a0f..9985c5d 100644 --- a/tests/verify_reference_synthesis.py +++ b/tests/verify_reference_synthesis.py @@ -64,6 +64,8 @@ def main() -> None: assert len(synthesis["source_tracks"]) == 3, synthesis assert synthesis["synthesis"]["borrow_now"], synthesis assert synthesis["synthesis"]["recommendation"]["summary"], synthesis + assert synthesis["synthesis"]["pattern_gate"]["accepted"], synthesis + assert synthesis["synthesis"]["pattern_gate"]["threshold"] == 4, synthesis assert synthesis["synthesis"]["visibility"]["mode"] == "explicit", synthesis assert "design_conflict" in synthesis["synthesis"]["visibility"]["reasons"], synthesis assert synthesis["synthesis"]["conflicts"], synthesis @@ -71,6 +73,7 @@ def main() -> None: assert "Curated World-Class Pattern Tracks" in markdown, markdown[:600] assert "Borrow Now" in markdown, markdown[:900] assert "Default Recommendation" in markdown, markdown[:1200] + assert "Pattern Gate" in markdown, markdown[:1500] assert "Conflict Check" in markdown, markdown[:1500] print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_review_viewer.py b/tests/verify_review_viewer.py index a236023..4c3d627 100644 --- a/tests/verify_review_viewer.py +++ b/tests/verify_review_viewer.py @@ -46,6 +46,8 @@ def main() -> None: assert "Architecture at a glance" in html_text, html_text[:500] assert "Compare view" in html_text, html_text[:500] assert "Variant diff studio" in html_text, html_text[:900] + assert "Evidence readiness" in html_text, html_text[:1200] + assert "Output risk profile" in html_text, html_text[:1600] assert "Reference coach" in html_text, html_text[:900] assert "Reference synthesis" in html_text, html_text[:1200] assert "Top three next moves" in html_text, html_text[:500] diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 76cd42d..9164dd1 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import json +import os import subprocess import sys from pathlib import Path @@ -11,12 +12,17 @@ BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan" def run(*args: str, input_text: str | None = None) -> dict: + env = { + **os.environ, + "YAO_UPDATE_VERSION_URL": (ROOT / "tests" / "tmp_cli" / "remote-version.txt").as_uri(), + } proc = subprocess.run( [sys.executable, str(CLI), *args], cwd=ROOT, capture_output=True, text=True, input=input_text, + env=env, ) payload = json.loads(proc.stdout) return { @@ -32,6 +38,8 @@ def main() -> None: if tmp_root.exists(): subprocess.run(["rm", "-rf", str(tmp_root)], check=True) tmp_root.mkdir(parents=True, exist_ok=True) + remote_version = tmp_root / "remote-version.txt" + remote_version.write_text("9.9.9\n", encoding="utf-8") init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root)) assert init_result["ok"], init_result @@ -44,7 +52,9 @@ def main() -> None: assert (created / "reports" / "review-viewer.html").exists(), created assert (created / "reports" / "reference-scan.md").exists(), created assert (created / "reports" / "reference-synthesis.md").exists(), created + assert (created / "reports" / "output-risk-profile.md").exists(), created assert (created / "reports" / "iteration-directions.md").exists(), created + assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created quickstart_result = run( "quickstart", @@ -74,6 +84,8 @@ def main() -> None: assert (quickstart_root / "reports" / "reference-synthesis.md").exists(), quickstart_root assert quickstart_result["payload"]["archetype"] == "production", quickstart_result assert quickstart_result["payload"]["guidance"]["experience_note"], quickstart_result + assert quickstart_result["payload"]["guidance"]["problem_diagnosis"]["candidates"], quickstart_result + assert "Update available for yao-meta-skill" in quickstart_result["stderr"], quickstart_result["stderr"] assert quickstart_result["payload"]["intent_confidence"]["score"] >= 70, quickstart_result assert quickstart_result["payload"]["recommendation"]["summary"], quickstart_result assert quickstart_result["payload"]["reference_mode"]["mode"] == "silent", quickstart_result @@ -155,6 +167,11 @@ def main() -> None: assert reference_synthesis_result["ok"], reference_synthesis_result assert reference_synthesis_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-synthesis.md"), reference_synthesis_result + output_risk_result = run("output-risk-profile", str(created)) + assert output_risk_result["ok"], output_risk_result + assert output_risk_result["payload"]["artifacts"]["markdown"].endswith("reports/output-risk-profile.md"), output_risk_result + assert output_risk_result["payload"]["summary"]["risk_families"], output_risk_result + directions_result = run("iteration-directions", str(created)) assert directions_result["ok"], directions_result assert directions_result["payload"]["artifacts"]["markdown"].endswith("reports/iteration-directions.md"), directions_result @@ -208,6 +225,17 @@ def main() -> None: assert package_result["ok"], package_result assert (package_dir / "targets" / "generic" / "adapter.json").exists(), package_dir + update_result = run( + "check-update", + "--force", + "--no-cache", + "--version-url", + remote_version.as_uri(), + ) + assert update_result["ok"], update_result + assert update_result["payload"]["update_available"], update_result + assert update_result["payload"]["remote_version"] == "9.9.9", update_result + test_result = run("test", "--target", "promotion-check") assert test_result["ok"], test_result