frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: add artifact design quality profiles

Browse Source
This commit is contained in:
yaojingang
2026-05-01 14:27:51 +08:00
parent e3ca33c962
commit 98d2593647
46 changed files with 1649 additions and 740 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+6 -3
View File
@@ -1,6 +1,6 @@
PYTHON ?= python3
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test ci-test clean
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test ci-test clean
eval:
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
@@ -56,6 +56,9 @@ reference-synthesis-check:
output-risk-profile-check:
$(PYTHON) tests/verify_output_risk_profile.py
artifact-design-profile-check:
$(PYTHON) tests/verify_artifact_design_profile.py
iteration-directions-check:
$(PYTHON) tests/verify_iteration_directions.py
@@ -107,10 +110,10 @@ resource-boundary-check:
quality-check:
$(PYTHON) tests/verify_quality_checks.py
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check
ci-test:
$(PYTHON) scripts/ci_test.py
clean:
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
README.md
+3 -1
View File
@@ -23,6 +23,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
- a silent-by-default GitHub benchmark scan plus reference synthesis that studies top public repositories and world-class pattern tracks, then surfaces only real conflicts or uncertainty to the user
- a generated visual HTML overview for each newly initialized skill
- a side-by-side HTML review studio for first-pass human review
- an artifact design profile that defines visual direction, layout patterns, and quality gates for reports, tutorials, dashboards, screenshots, and review pages
- three high-value next iteration directions after the first package is created
- a lightweight feedback log that does not require a full promotion cycle
- a baseline compare report for with-skill vs baseline review
@@ -81,7 +82,7 @@ Weighted score formula: `sum(score / 10 * weight)`.
2. Start with a short, human intent dialogue so the real job, outputs, exclusions, constraints, and standards are explicit.
3. Let `quickstart` clarify intent first, then run silent benchmark scan and reference synthesis; it only surfaces explicit questions when intent is still unclear or when there is a real design conflict.
4. Use the archetype-aware `quickstart` or the full authoring flow to generate or improve the package in scaffold, production, library, or governed mode.
5. Review the generated `reports/intent-dialogue.md`, `reports/intent-confidence.md`, `reports/reference-synthesis.md`, `reports/skill-overview.html`, and `reports/iteration-directions.md` before adding more structure.
5. Review the generated `reports/intent-dialogue.md`, `reports/intent-confidence.md`, `reports/reference-synthesis.md`, `reports/artifact-design-profile.md`, `reports/skill-overview.html`, and `reports/iteration-directions.md` before adding more structure.
Or use the unified authoring CLI:
@@ -93,6 +94,7 @@ python3 scripts/yao.py reference-scan my-skill \
--user-reference "A product or repo I admire::taste::Learn the clarity and operating standard.::Do not copy wording." \
--local-constraint "Current Library Naming::structure::Keep naming aligned with the local skill library.::Do not inherit private references."
python3 scripts/yao.py review-viewer my-skill
python3 scripts/yao.py artifact-design-profile my-skill
python3 scripts/yao.py feedback my-skill --note "Tighten exclusions before adding scripts." --rating 4 --category boundary
python3 scripts/yao.py baseline-compare
python3 scripts/yao.py check-update
SKILL.md
+7 -6
View File
@@ -23,18 +23,19 @@ Build reusable skill packages, not long prompts.
- `Production`: team reuse with focused gates.
- `Library`: shared infrastructure or meta skill.
Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md), [Authoring Discipline](references/authoring-discipline.md).
Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).
## Compact Workflow
1. Decide whether the request should become a skill and choose the lightest fit.
2. Run a short intent dialogue to capture the job, output, exclusions, constraints, and standards.
2. Capture job, output, exclusions, constraints, and standards.
3. Run a reference scan: external benchmarks first, user references second, local fit third. Keep synthesis silent unless intent stays unclear or a real design conflict needs a user call.
4. Write the `description` early and test route quality before expanding the package.
5. Add only the folders and gates that earn their keep.
6. After the first package exists, surface the top three next iteration directions.
5. Add output-risk and artifact-design profiles when user-facing deliverables matter.
6. Add only folders and gates that earn their keep.
7. Surface the top three next iteration directions.
Core playbooks: [Method](references/skill-engineering-method.md), [Intent Dialogue](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Authoring Discipline](references/authoring-discipline.md).
Core playbooks: [Method](references/skill-engineering-method.md), [Intent Dialogue](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Artifact Design](references/artifact-design-doctrine.md).
## First-Turn Style
@@ -62,4 +63,4 @@ Unless the user asks otherwise, produce:
## Reference Map
Primary references: [Method](references/skill-engineering-method.md), [Authoring Discipline](references/authoring-discipline.md), [Reference Scan](references/reference-scan.md), [Intent Dialogue](references/intent-dialogue.md), [Governance](references/governance.md), [Resource Boundaries](references/resource-boundaries.md).
Primary references: [Method](references/skill-engineering-method.md), [Reference Scan](references/reference-scan.md), [Artifact Design](references/artifact-design-doctrine.md), [Output Visual Quality](references/output-visual-quality.md), [Governance](references/governance.md).
examples/governed-incident-command/optimization/reports/description_optimization.json
+68 -68
View File
@@ -394,7 +394,7 @@
{
"prompt": "Summarize this incident but do not package it as a reusable skill.",
"family": "summary_only",
"score": 0.063,
"score": 0.07,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -416,7 +416,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.04,
"support_score": 0,
"lexical_support": 0.333,
"lexical_support": 0.417,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -461,7 +461,7 @@
],
"semantic_coverage": 0.14,
"support_score": 0,
"lexical_support": 0.286,
"lexical_support": 0.429,
"negative_penalty": 0.24,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -557,7 +557,7 @@
"bucket": "should_trigger",
"family": "governed_workflow",
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
"score": 0.394,
"score": 0.399,
"reason": "false_negative",
"matched_desired_concepts": [
"governed_workflow",
@@ -584,7 +584,7 @@
{
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
"family": "governed_workflow",
"score": 0.394,
"score": 0.399,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -608,7 +608,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.32,
"support_score": 0,
"lexical_support": 0.5,
"lexical_support": 0.562,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -732,7 +732,7 @@
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"lexical_support": 0.111,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -750,7 +750,7 @@
{
"prompt": "Draft a one-off internal status update from these incident notes.",
"family": "one_off_update",
"score": 0.216,
"score": 0.22,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -772,7 +772,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.2,
"support_score": 0,
"lexical_support": 0.4,
"lexical_support": 0.455,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1258,7 +1258,7 @@
],
"semantic_coverage": 0.227,
"support_score": 0.04,
"lexical_support": 0.214,
"lexical_support": 0.286,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1365,7 +1365,7 @@
{
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
"family": "governed_workflow",
"score": 0.655,
"score": 0.66,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -1387,7 +1387,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.636,
"support_score": 0.04,
"lexical_support": 0.25,
"lexical_support": 0.312,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -1519,7 +1519,7 @@
{
"prompt": "Draft a one-off internal status update from these incident notes.",
"family": "one_off_update",
"score": 0.058,
"score": 0.057,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -1538,7 +1538,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.2,
"lexical_support": 0.1,
"lexical_support": 0.091,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1988,7 +1988,7 @@
],
"semantic_coverage": 0.227,
"support_score": 0.04,
"lexical_support": 0.214,
"lexical_support": 0.286,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -2095,7 +2095,7 @@
{
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
"family": "governed_workflow",
"score": 0.655,
"score": 0.66,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2117,7 +2117,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.636,
"support_score": 0.04,
"lexical_support": 0.25,
"lexical_support": 0.312,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2249,7 +2249,7 @@
{
"prompt": "Draft a one-off internal status update from these incident notes.",
"family": "one_off_update",
"score": 0.058,
"score": 0.057,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -2268,7 +2268,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.2,
"lexical_support": 0.1,
"lexical_support": 0.091,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -2435,7 +2435,7 @@
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"score": 0.798,
"score": 0.804,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2457,7 +2457,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.773,
"support_score": 0.04,
"lexical_support": 0.467,
"lexical_support": 0.533,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2754,7 +2754,7 @@
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"score": 0.798,
"score": 0.804,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2776,7 +2776,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.773,
"support_score": 0.04,
"lexical_support": 0.467,
"lexical_support": 0.533,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -3024,7 +3024,7 @@
"bucket": "should_trigger",
"family": "blind_governed_workflow",
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"score": 0.452,
"score": 0.458,
"reason": "false_negative",
"matched_desired_concepts": [
"comms_and_owners",
@@ -3089,7 +3089,7 @@
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"score": 0.452,
"score": 0.458,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -3113,7 +3113,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.38,
"support_score": 0,
"lexical_support": 0.533,
"lexical_support": 0.6,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -3212,7 +3212,7 @@
{
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
"family": "blind_summary_only",
"score": 0.387,
"score": 0.395,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -3236,7 +3236,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.34,
"support_score": 0,
"lexical_support": 0.182,
"lexical_support": 0.273,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -5246,7 +5246,7 @@
{
"prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.",
"family": "adversarial_future_playbook_not_packet",
"score": 0.065,
"score": 0.071,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -5268,7 +5268,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.04,
"support_score": 0,
"lexical_support": 0.357,
"lexical_support": 0.429,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -5422,9 +5422,9 @@
],
"holdout": {
"false_positives": 0,
"false_negatives": 2,
"precision": null,
"recall": 0.0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
@@ -5656,45 +5656,45 @@
},
"winner_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.367,
"mean_positive_score": 0.37,
"mean_non_trigger_score": 0.019,
"mean_near_neighbor_score": 0.038,
"mean_near_neighbor_score": 0.037,
"min_positive_score": 0.08,
"max_non_trigger_score": 0.058,
"score_gap": 0.022,
"max_non_trigger_score": 0.057,
"score_gap": 0.023,
"threshold_margin": -0.4,
"positive_threshold_buffer": -0.4,
"negative_threshold_buffer": 0.422,
"negative_threshold_buffer": 0.423,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "overlap"
},
"current_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.367,
"mean_positive_score": 0.37,
"mean_non_trigger_score": 0.019,
"mean_near_neighbor_score": 0.038,
"mean_near_neighbor_score": 0.037,
"min_positive_score": 0.08,
"max_non_trigger_score": 0.058,
"score_gap": 0.022,
"max_non_trigger_score": 0.057,
"score_gap": 0.023,
"threshold_margin": -0.4,
"positive_threshold_buffer": -0.4,
"negative_threshold_buffer": 0.422,
"negative_threshold_buffer": 0.423,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "overlap"
},
"baseline_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.358,
"mean_non_trigger_score": 0.058,
"mean_near_neighbor_score": 0.117,
"mean_positive_score": 0.36,
"mean_non_trigger_score": 0.059,
"mean_near_neighbor_score": 0.119,
"min_positive_score": 0.321,
"max_non_trigger_score": 0.216,
"score_gap": 0.105,
"max_non_trigger_score": 0.22,
"score_gap": 0.101,
"threshold_margin": -0.159,
"positive_threshold_buffer": -0.159,
"negative_threshold_buffer": 0.264,
"negative_threshold_buffer": 0.26,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
"risk_band": "overlap"
@@ -5780,14 +5780,14 @@
},
"winner_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.899,
"mean_positive_score": 0.902,
"mean_non_trigger_score": 0.168,
"mean_near_neighbor_score": 0.138,
"min_positive_score": 0.798,
"min_positive_score": 0.804,
"max_non_trigger_score": 0.386,
"score_gap": 0.412,
"score_gap": 0.418,
"threshold_margin": 0.094,
"positive_threshold_buffer": 0.318,
"positive_threshold_buffer": 0.324,
"negative_threshold_buffer": 0.094,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -5795,14 +5795,14 @@
},
"current_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.899,
"mean_positive_score": 0.902,
"mean_non_trigger_score": 0.168,
"mean_near_neighbor_score": 0.138,
"min_positive_score": 0.798,
"min_positive_score": 0.804,
"max_non_trigger_score": 0.386,
"score_gap": 0.412,
"score_gap": 0.418,
"threshold_margin": 0.094,
"positive_threshold_buffer": 0.318,
"positive_threshold_buffer": 0.324,
"negative_threshold_buffer": 0.094,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -5810,15 +5810,15 @@
},
"baseline_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.622,
"mean_non_trigger_score": 0.149,
"mean_near_neighbor_score": 0.194,
"min_positive_score": 0.452,
"max_non_trigger_score": 0.387,
"score_gap": 0.065,
"threshold_margin": -0.028,
"positive_threshold_buffer": -0.028,
"negative_threshold_buffer": 0.093,
"mean_positive_score": 0.625,
"mean_non_trigger_score": 0.151,
"mean_near_neighbor_score": 0.198,
"min_positive_score": 0.458,
"max_non_trigger_score": 0.395,
"score_gap": 0.063,
"threshold_margin": -0.022,
"positive_threshold_buffer": -0.022,
"negative_threshold_buffer": 0.085,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "overlap"
@@ -5992,8 +5992,8 @@
"baseline_calibration": {
"threshold": 0.34,
"mean_positive_score": 0.689,
"mean_non_trigger_score": 0.073,
"mean_near_neighbor_score": 0.033,
"mean_non_trigger_score": 0.074,
"mean_near_neighbor_score": 0.035,
"min_positive_score": 0.387,
"max_non_trigger_score": 0.215,
"score_gap": 0.172,
examples/governed-incident-command/optimization/reports/description_optimization.md
+3 -3
View File
@@ -16,7 +16,7 @@ Build governed incident command packets. Use when asked to standardize incident
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| `Current` | 37 | 0 | 1 | 1.0 | 0 | 1 |
| `Guardrail` | 51 | 0 | 1 | 1.0 | 1 | 1 |
| `Balanced` | 54 | 0 | 1 | 1.0 | 0 | 2 |
| `Balanced` | 54 | 0 | 1 | 1.0 | 0 | 1 |
| `Boundary` | 78 | 0 | 1 | 1.0 | 0 | 1 |
| `Artifact Aware` | 78 | 0 | 1 | 1.0 | 0 | 1 |
@@ -33,8 +33,8 @@ Build governed incident command packets. Use when asked to standardize incident
| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |
| --- | ---: | --- | ---: | ---: | ---: |
| Holdout | 0.022 | overlap | 0.0 | 0.022 | 0.105 |
| Blind Holdout | 0.412 | healthy | 0.167 | 0.412 | 0.065 |
| Holdout | 0.023 | overlap | 0.0 | 0.023 | 0.101 |
| Blind Holdout | 0.418 | healthy | 0.167 | 0.418 | 0.063 |
| Adversarial Holdout | 0.598 | healthy | 0.0 | 0.598 | 0.172 |
## Judge Blind Summary
examples/team-frontend-review/optimization/reports/description_optimization.json
+110 -123
View File
@@ -39,14 +39,14 @@
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 4,
"precision": null,
"recall": 0.0,
"false_negatives": 3,
"precision": 1.0,
"recall": 0.25,
"bucket_stats": {
"should_trigger": {
"total": 4,
"passed": 0,
"pass_rate": 0.0
"passed": 1,
"pass_rate": 0.25
},
"should_not_trigger": {
"total": 4,
@@ -76,10 +76,10 @@
},
"security_and_a11y": {
"total": 1,
"passed": 0,
"passed": 1,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.0
"false_negatives": 0,
"pass_rate": 1.0
},
"build_only": {
"total": 1,
@@ -136,7 +136,7 @@
"bucket": "should_trigger",
"family": "premerge_review",
"prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.",
"score": 0.31,
"score": 0.316,
"reason": "false_negative",
"matched_desired_concepts": [
"accessibility",
@@ -148,7 +148,7 @@
"bucket": "should_trigger",
"family": "state_review",
"prompt": "Do a frontend review on these UI components and check loading, error, and empty states.",
"score": 0.42,
"score": 0.424,
"reason": "false_negative",
"matched_desired_concepts": [
"frontend_scope",
@@ -156,24 +156,11 @@
],
"matched_negative_concepts": []
},
{
"bucket": "should_trigger",
"family": "security_and_a11y",
"prompt": "Audit this signup form for a11y, unsafe rendering, and UX regressions.",
"score": 0.478,
"reason": "false_negative",
"matched_desired_concepts": [
"accessibility",
"frontend_scope",
"ui_security"
],
"matched_negative_concepts": []
},
{
"bucket": "should_trigger",
"family": "premerge_review",
"prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.",
"score": 0.181,
"score": 0.188,
"reason": "false_negative",
"matched_desired_concepts": [
"frontend_scope",
@@ -187,7 +174,7 @@
{
"prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.",
"family": "premerge_review",
"score": 0.31,
"score": 0.316,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -210,7 +197,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.24,
"support_score": 0,
"lexical_support": 0.615,
"lexical_support": 0.692,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -231,7 +218,7 @@
{
"prompt": "Do a frontend review on these UI components and check loading, error, and empty states.",
"family": "state_review",
"score": 0.42,
"score": 0.424,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -254,7 +241,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.38,
"support_score": 0,
"lexical_support": 0.385,
"lexical_support": 0.429,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -275,10 +262,10 @@
{
"prompt": "Audit this signup form for a11y, unsafe rendering, and UX regressions.",
"family": "security_and_a11y",
"score": 0.478,
"predicted_trigger": false,
"score": 0.483,
"predicted_trigger": true,
"expected_trigger": true,
"passed": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
@@ -299,7 +286,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.42,
"support_score": 0,
"lexical_support": 0.4,
"lexical_support": 0.455,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -322,7 +309,7 @@
{
"prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.",
"family": "premerge_review",
"score": 0.181,
"score": 0.188,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -345,7 +332,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.12,
"support_score": 0,
"lexical_support": 0.385,
"lexical_support": 0.467,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -564,7 +551,7 @@
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.273,
"lexical_support": 0.333,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -736,7 +723,7 @@
{
"prompt": "Give this React modal a pre-merge frontend review for a11y, risky rendering, and missing states.",
"family": "premerge_review",
"score": 0.868,
"score": 0.869,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -762,7 +749,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.82,
"support_score": 0,
"lexical_support": 0.667,
"lexical_support": 0.688,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -792,7 +779,7 @@
{
"prompt": "Review this UI diff for accessibility gaps, unsafe form handling, and UX regressions.",
"family": "security_and_a11y",
"score": 0.483,
"score": 0.489,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -816,7 +803,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.42,
"support_score": 0,
"lexical_support": 0.455,
"lexical_support": 0.538,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -920,7 +907,7 @@
{
"prompt": "Discuss possible UI improvements for this checkout flow, but do not inspect the code yet.",
"family": "brainstorm_only",
"score": 0.048,
"score": 0.053,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -942,7 +929,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.04,
"support_score": 0,
"lexical_support": 0.143,
"lexical_support": 0.2,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1131,7 +1118,7 @@
"bucket": "should_trigger",
"family": "premerge_review",
"prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.",
"score": 0.298,
"score": 0.304,
"reason": "false_negative",
"matched_desired_concepts": [
"accessibility",
@@ -1143,7 +1130,7 @@
"bucket": "should_trigger",
"family": "state_review",
"prompt": "Do a frontend review on these UI components and check loading, error, and empty states.",
"score": 0.433,
"score": 0.435,
"reason": "false_negative",
"matched_desired_concepts": [
"frontend_scope",
@@ -1155,7 +1142,7 @@
"bucket": "should_trigger",
"family": "premerge_review",
"prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.",
"score": 0.181,
"score": 0.188,
"reason": "false_negative",
"matched_desired_concepts": [
"frontend_scope",
@@ -1169,7 +1156,7 @@
{
"prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.",
"family": "premerge_review",
"score": 0.298,
"score": 0.304,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -1192,7 +1179,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.24,
"support_score": 0,
"lexical_support": 0.462,
"lexical_support": 0.538,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -1213,7 +1200,7 @@
{
"prompt": "Do a frontend review on these UI components and check loading, error, and empty states.",
"family": "state_review",
"score": 0.433,
"score": 0.435,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -1236,7 +1223,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.38,
"support_score": 0,
"lexical_support": 0.538,
"lexical_support": 0.571,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -1257,7 +1244,7 @@
{
"prompt": "Audit this signup form for a11y, unsafe rendering, and UX regressions.",
"family": "security_and_a11y",
"score": 0.494,
"score": 0.497,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -1281,7 +1268,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.42,
"support_score": 0,
"lexical_support": 0.6,
"lexical_support": 0.636,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -1304,7 +1291,7 @@
{
"prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.",
"family": "premerge_review",
"score": 0.181,
"score": 0.188,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -1327,7 +1314,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.12,
"support_score": 0,
"lexical_support": 0.385,
"lexical_support": 0.467,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -1546,7 +1533,7 @@
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.273,
"lexical_support": 0.333,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1718,7 +1705,7 @@
{
"prompt": "Give this React modal a pre-merge frontend review for a11y, risky rendering, and missing states.",
"family": "premerge_review",
"score": 0.873,
"score": 0.874,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -1744,7 +1731,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.82,
"support_score": 0,
"lexical_support": 0.733,
"lexical_support": 0.75,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -1774,7 +1761,7 @@
{
"prompt": "Review this UI diff for accessibility gaps, unsafe form handling, and UX regressions.",
"family": "security_and_a11y",
"score": 0.49,
"score": 0.496,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -1798,7 +1785,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.42,
"support_score": 0,
"lexical_support": 0.545,
"lexical_support": 0.615,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -1902,7 +1889,7 @@
{
"prompt": "Discuss possible UI improvements for this checkout flow, but do not inspect the code yet.",
"family": "brainstorm_only",
"score": 0.048,
"score": 0.053,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -1924,7 +1911,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.04,
"support_score": 0,
"lexical_support": 0.143,
"lexical_support": 0.2,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -2080,7 +2067,7 @@
"bucket": "should_trigger",
"family": "premerge_review",
"prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.",
"score": 0.298,
"score": 0.304,
"reason": "false_negative",
"matched_desired_concepts": [
"accessibility",
@@ -2092,7 +2079,7 @@
"bucket": "should_trigger",
"family": "state_review",
"prompt": "Do a frontend review on these UI components and check loading, error, and empty states.",
"score": 0.433,
"score": 0.435,
"reason": "false_negative",
"matched_desired_concepts": [
"frontend_scope",
@@ -2104,7 +2091,7 @@
"bucket": "should_trigger",
"family": "premerge_review",
"prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.",
"score": 0.181,
"score": 0.188,
"reason": "false_negative",
"matched_desired_concepts": [
"frontend_scope",
@@ -2118,7 +2105,7 @@
{
"prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.",
"family": "premerge_review",
"score": 0.298,
"score": 0.304,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -2141,7 +2128,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.24,
"support_score": 0,
"lexical_support": 0.462,
"lexical_support": 0.538,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2162,7 +2149,7 @@
{
"prompt": "Do a frontend review on these UI components and check loading, error, and empty states.",
"family": "state_review",
"score": 0.433,
"score": 0.435,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -2185,7 +2172,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.38,
"support_score": 0,
"lexical_support": 0.538,
"lexical_support": 0.571,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2206,7 +2193,7 @@
{
"prompt": "Audit this signup form for a11y, unsafe rendering, and UX regressions.",
"family": "security_and_a11y",
"score": 0.494,
"score": 0.497,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2230,7 +2217,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.42,
"support_score": 0,
"lexical_support": 0.6,
"lexical_support": 0.636,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -2253,7 +2240,7 @@
{
"prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.",
"family": "premerge_review",
"score": 0.181,
"score": 0.188,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
@@ -2276,7 +2263,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.12,
"support_score": 0,
"lexical_support": 0.385,
"lexical_support": 0.467,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2495,7 +2482,7 @@
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.273,
"lexical_support": 0.333,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -2667,7 +2654,7 @@
{
"prompt": "Give this React modal a pre-merge frontend review for a11y, risky rendering, and missing states.",
"family": "premerge_review",
"score": 0.873,
"score": 0.874,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2693,7 +2680,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.82,
"support_score": 0,
"lexical_support": 0.733,
"lexical_support": 0.75,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -2723,7 +2710,7 @@
{
"prompt": "Review this UI diff for accessibility gaps, unsafe form handling, and UX regressions.",
"family": "security_and_a11y",
"score": 0.49,
"score": 0.496,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2747,7 +2734,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.42,
"support_score": 0,
"lexical_support": 0.545,
"lexical_support": 0.615,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -2851,7 +2838,7 @@
{
"prompt": "Discuss possible UI improvements for this checkout flow, but do not inspect the code yet.",
"family": "brainstorm_only",
"score": 0.048,
"score": 0.053,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -2873,7 +2860,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.04,
"support_score": 0,
"lexical_support": 0.143,
"lexical_support": 0.2,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -3049,7 +3036,7 @@
{
"prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.",
"family": "blind_premerge",
"score": 0.859,
"score": 0.866,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -3075,7 +3062,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.82,
"support_score": 0,
"lexical_support": 0.562,
"lexical_support": 0.647,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -3387,7 +3374,7 @@
{
"prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.",
"family": "blind_premerge",
"score": 0.859,
"score": 0.866,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -3413,7 +3400,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.82,
"support_score": 0,
"lexical_support": 0.562,
"lexical_support": 0.647,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -3725,7 +3712,7 @@
{
"prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.",
"family": "blind_premerge",
"score": 0.854,
"score": 0.857,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -3751,7 +3738,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.82,
"support_score": 0,
"lexical_support": 0.5,
"lexical_support": 0.529,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -4083,7 +4070,7 @@
],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 6,
"capability_overlap": 7,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
@@ -4215,7 +4202,7 @@
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"capability_overlap": 3,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
@@ -4421,7 +4408,7 @@
],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 6,
"capability_overlap": 7,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
@@ -4553,7 +4540,7 @@
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"capability_overlap": 3,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
@@ -4759,7 +4746,7 @@
],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 6,
"capability_overlap": 7,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
@@ -4891,7 +4878,7 @@
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"capability_overlap": 3,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
@@ -5056,7 +5043,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.84,
"support_score": 0,
"lexical_support": 0.545,
"lexical_support": 0.542,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -5440,7 +5427,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.84,
"support_score": 0,
"lexical_support": 0.545,
"lexical_support": 0.542,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -5798,7 +5785,7 @@
{
"prompt": "Ignore the feature-request chatter and run a pre-merge frontend review on this React diff for keyboard navigation, unsafe rendering, and missing empty states.",
"family": "adversarial_noisy_premerge_review",
"score": 0.862,
"score": 0.863,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -5824,7 +5811,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.84,
"support_score": 0,
"lexical_support": 0.364,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -6320,14 +6307,14 @@
},
"winner_vs_baseline_dev": {
"baseline_false_positives": 0,
"baseline_false_negatives": 4,
"baseline_false_negatives": 3,
"improved_false_positives": 0,
"improved_false_negatives": 3,
"false_positive_delta": 0,
"false_negative_delta": -1,
"baseline_precision": null,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 0.0,
"baseline_recall": 0.25,
"improved_recall": 0.25
},
"winner_vs_baseline_holdout": {
@@ -6432,14 +6419,14 @@
},
"winner_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_non_trigger_score": 0.088,
"mean_near_neighbor_score": 0.042,
"min_positive_score": 0.49,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.089,
"mean_near_neighbor_score": 0.045,
"min_positive_score": 0.496,
"max_non_trigger_score": 0.211,
"score_gap": 0.279,
"threshold_margin": 0.01,
"positive_threshold_buffer": 0.01,
"score_gap": 0.285,
"threshold_margin": 0.016,
"positive_threshold_buffer": 0.016,
"negative_threshold_buffer": 0.269,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -6447,14 +6434,14 @@
},
"current_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_non_trigger_score": 0.088,
"mean_near_neighbor_score": 0.042,
"min_positive_score": 0.49,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.089,
"mean_near_neighbor_score": 0.045,
"min_positive_score": 0.496,
"max_non_trigger_score": 0.211,
"score_gap": 0.279,
"threshold_margin": 0.01,
"positive_threshold_buffer": 0.01,
"score_gap": 0.285,
"threshold_margin": 0.016,
"positive_threshold_buffer": 0.016,
"negative_threshold_buffer": 0.269,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -6462,14 +6449,14 @@
},
"baseline_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.675,
"mean_non_trigger_score": 0.086,
"mean_near_neighbor_score": 0.042,
"min_positive_score": 0.483,
"mean_positive_score": 0.679,
"mean_non_trigger_score": 0.087,
"mean_near_neighbor_score": 0.045,
"min_positive_score": 0.489,
"max_non_trigger_score": 0.211,
"score_gap": 0.272,
"threshold_margin": 0.003,
"positive_threshold_buffer": 0.003,
"score_gap": 0.278,
"threshold_margin": 0.009,
"positive_threshold_buffer": 0.009,
"negative_threshold_buffer": 0.269,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -6533,7 +6520,7 @@
},
"winner_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.177,
"mean_near_neighbor_score": 0.226,
"min_positive_score": 0.504,
@@ -6548,7 +6535,7 @@
},
"current_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.177,
"mean_near_neighbor_score": 0.226,
"min_positive_score": 0.504,
@@ -6563,7 +6550,7 @@
},
"baseline_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.673,
"mean_positive_score": 0.674,
"mean_non_trigger_score": 0.177,
"mean_near_neighbor_score": 0.223,
"min_positive_score": 0.492,
@@ -6738,7 +6725,7 @@
},
"baseline_calibration": {
"threshold": 0.34,
"mean_positive_score": 0.703,
"mean_positive_score": 0.704,
"mean_non_trigger_score": 0.006,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.544,
@@ -6802,7 +6789,7 @@
"winner_adversarial_score_gap": 0.509,
"candidate_count": 5,
"baseline_tokens": 52,
"baseline_dev_total_errors": 4,
"baseline_dev_total_errors": 3,
"baseline_holdout_total_errors": 0,
"baseline_blind_holdout_total_errors": 0,
"baseline_judge_blind_holdout_total_errors": 0,
examples/team-frontend-review/optimization/reports/description_optimization.md
+1 -1
View File
@@ -33,7 +33,7 @@ Review frontend code for accessibility, UI security, missing states, and UX regr
| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |
| --- | ---: | --- | ---: | ---: | ---: |
| Holdout | 0.279 | tight | 0.167 | 0.279 | 0.272 |
| Holdout | 0.285 | tight | 0.167 | 0.285 | 0.278 |
| Blind Holdout | 0.088 | tight | 0.333 | 0.088 | 0.082 |
| Adversarial Holdout | 0.509 | healthy | 0.0 | 0.509 | 0.519 |
references/artifact-design-doctrine.md
+49
View File
@@ -0,0 +1,49 @@
# Artifact Design Doctrine
Use this layer when a skill produces user-facing artifacts: HTML reports, Markdown tutorials, review viewers, dashboards, screenshots, tables, slide-like pages, or generated skill overview pages.
## Principle
Output quality is part of skill quality. A generated skill should not only know what to do; it should also know how its final artifact should read, scan, and hold up under review.
The visual system must follow the artifact's purpose. Do not inherit a fixed house style just because a reference skill uses one. For example, Kami's paper-editorial discipline is useful, but its warm parchment background is not a default requirement for Yao-generated artifacts.
## What To Borrow From Document Skills
- route by artifact type before designing the page
- extract facts, claims, numbers, actions, and missing inputs before formatting
- ask one focused question when the artifact lacks a necessary audience, input, or output standard
- keep prose, layout, and production checks separated
- verify placeholders, headings, paths, screenshots, and render-critical assumptions before delivery
## What To Borrow From Presentation Skills
- plan the artifact's role, hierarchy, density, rhythm, and evidence before writing HTML
- choose a concrete visual direction from the topic, not from a generic AI template
- use structure, spacing, type, and contrast before decoration
- split dense content instead of squeezing it into one surface
- reject generic purple gradients, glass cards, repeated card grids, and decorative screenshots
## Content-Led Visual Direction
Pick the design system from the work:
- high-trust reports: restrained editorial layout, strong hierarchy, compact evidence blocks
- tutorials: clear progressive sections, success checks, screenshots only when real and necessary
- dashboards: compact metrics, visible deltas, short explanations, no paragraph-heavy tables
- review viewers: side-by-side comparison, reviewer-visible evidence, explicit tradeoffs
- slide-like artifacts: rhythm, section breaks, big claims, controlled density
## Non-Negotiables
- headings must be specific to the user's domain and outcome
- tables are used only when comparison is the main job
- citations and footnotes must not interrupt ordinary reading
- screenshots and visual evidence must be real, sourceable, and correctly described
- final HTML must not contain absolute local filesystem paths
- mobile and narrow-width reading must remain usable
- design tokens must be named and coherent: type, color, spacing, surface, and emphasis
## Reviewer Rule
Reviewers should see both the artifact's intended visual direction and the top risks that could make the output feel low quality. If a report looks polished but hides weak headings, bad tables, wrong screenshots, or citation clutter, the skill is not ready.
references/output-quality-risk.md
+4 -3
View File
@@ -21,9 +21,10 @@ Before finalizing a generated skill:
1. infer the most likely output families from the job and target output
2. generate `reports/output-risk-profile.md`
3. add output-specific constraints to the generated skill's operating frame
4. expose the risk profile in the review viewer
5. treat unresolved output risks as iteration candidates instead of pretending the first version is complete
3. generate `reports/artifact-design-profile.md` when the output is a report, tutorial, viewer, dashboard, screenshot, Markdown page, or visual artifact
4. add output-specific constraints to the generated skill's operating frame
5. expose the risk and design profiles in the review viewer
6. treat unresolved output risks as iteration candidates instead of pretending the first version is complete
## Self-Repair Rule
references/output-visual-quality.md
+53
View File
@@ -0,0 +1,53 @@
# Output Visual Quality
Use this checklist before approving a generated skill that produces reports, tutorials, HTML pages, screenshots, Markdown deliverables, or slide-like artifacts.
## Common Visual Failures
- generic headings such as Overview, Key Points, Summary, or Next Steps when the user's domain needs sharper section names
- large citation or footnote clusters that break sentence flow
- Markdown tables with paragraph-length cells or weak hierarchy
- screenshots captured from the wrong state, viewport, crop, or missing asset
- HTML reports that look like raw JSON converted into cards
- repeated cards with identical weight, making the page hard to scan
- decorative gradients, shadows, or glass effects that do not serve the content
- mobile layouts that collapse into long undifferentiated blocks
## Design Quality Gates
### P0 Must Fix
- no absolute `/Users/...` paths in final HTML
- no placeholder titles, labels, screenshots, or source notes
- no invented screenshots, charts, citations, or visual evidence
- no table with paragraph-length cells when bullets or cards would scan better
- no fixed design palette copied from another skill without content justification
### P1 Should Fix
- title and section headings use domain nouns and the target outcome
- each report has one clear first-screen explanation of what it is for
- visual hierarchy separates decisions, evidence, risks, and next actions
- dense content is split across sections instead of squeezed into one block
- reviewer-only detail is present but not pushed into the user's main reading flow
### P2 Polish
- typography roles are consistent
- whitespace rhythm supports reading speed
- cards, tables, and callouts are used for different semantic jobs
- source notes are grouped where they preserve flow
- mobile, print, and static-file viewing are considered when relevant
## Self-Repair Pass
Before handoff, scan the generated artifact for:
1. heading specificity
2. table readability
3. citation density
4. screenshot truthfulness
5. local path leakage
6. placeholder remnants
7. mobile scanability
8. reviewer-visible evidence
references/skill-engineering-method.md
+18 -3
View File
@@ -11,9 +11,10 @@ This doctrine defines the default method for turning messy workflow material int
5. Write and test the trigger description before expanding the body.
6. Apply authoring discipline: name unresolved assumptions, keep scope small, and tie meaningful changes to checks.
7. Add an output risk profile for user-facing artifacts.
8. Add only the gates that match the risk.
9. Ship the first routeable package, then pick the three highest-value next iteration directions.
10. Package and govern the skill only as far as real reuse demands.
8. Add an artifact design profile for reports, tutorials, viewers, dashboards, screenshots, and visual pages.
9. Add only the gates that match the risk.
10. Ship the first routeable package, then pick the three highest-value next iteration directions.
11. Package and govern the skill only as far as real reuse demands.
## Phase 1: Qualification
@@ -122,6 +123,20 @@ Generate `reports/output-risk-profile.md` and expose it to the reviewer before a
See [Output Quality Risk](output-quality-risk.md).
## Phase 5.6: Artifact Design Profiling
Before approving generated reports or visual outputs, define how the artifact should read and scan.
- choose the artifact family: tutorial, report, review viewer, dashboard, visual evidence, slide-like narrative, or code/CLI guide
- let the content choose the visual system instead of copying a fixed house style
- borrow document discipline from Kami: route by document type, distill content, verify layout-critical assumptions
- borrow slide discipline from presentation skills: plan hierarchy, density, rhythm, and quality gates before writing HTML
- reject generic headings, noisy citations, weak tables, wrong screenshots, repeated card grids, and decorative visual defaults
Generate `reports/artifact-design-profile.md` and expose it in the overview and review viewer.
See [Artifact Design Doctrine](artifact-design-doctrine.md) and [Output Visual Quality](output-visual-quality.md).
## Phase 6: Trigger-First Authoring
Author the frontmatter `description` before expanding the body.
reports/artifact-design-profile.json
+82
View File
@@ -0,0 +1,82 @@
{
"skill_name": "yao-meta-skill",
"design_system": "content-led editorial",
"primary_artifact": {
"key": "report",
"label": "Report or brief",
"direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.",
"matched_keywords": [
"report",
"summary"
]
},
"artifact_families": [
{
"key": "report",
"label": "Report or brief",
"score": 2,
"matched_keywords": [
"report",
"summary"
],
"direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail."
},
{
"key": "code_or_cli",
"label": "Code, CLI, or implementation guide",
"score": 1,
"matched_keywords": [
"script"
],
"direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit."
}
],
"layout_patterns": [
"thesis",
"evidence blocks",
"decision table",
"risks",
"next actions",
"prerequisites",
"commands",
"expected output"
],
"design_tokens": {
"type": [
"Use a distinctive display face or serif for major claims when the artifact is editorial.",
"Use a restrained sans for dense body text and technical details.",
"Use mono only for metadata, paths, commands, labels, and evidence tags."
],
"color": [
"Choose colors from the artifact's domain, brand, or evidence mood.",
"Do not default to Kami parchment, purple gradients, or generic SaaS blue unless the content justifies it.",
"Keep accent color limited to decisions, active states, risk, or section anchors."
],
"spacing": [
"Prefer clear grid rhythm over floating decorative cards.",
"Increase whitespace around decisions and shrink whitespace around supporting metadata.",
"Split dense content instead of shrinking type or adding scroll traps."
],
"components": [
"Use cards for grouped evidence, tables for comparisons, callouts for decisions, and timelines for sequence.",
"Avoid cards inside cards.",
"Keep reviewer-only detail visible but visually quieter than user-facing guidance."
]
},
"quality_gates": [
"Keep the first screen useful without requiring the reader to parse every detail.",
"Use tables only for comparisons; move explanations below the table.",
"Keep source notes readable without flooding the body with markers.",
"Name the working directory and required inputs for commands.",
"Mark destructive, networked, or external side-effect operations.",
"Prefer the smallest runnable snippet over broad framework scaffolding."
],
"anti_patterns": [
"Do not copy Kami's fixed parchment background as a default.",
"Do not use generic purple gradients, glass cards, or stock SaaS hero sections unless the content calls for them.",
"Do not let Markdown tables become the default shape for every comparison or explanation.",
"Do not turn reviewer evidence into user-facing clutter.",
"Do not invent screenshots, citations, charts, or UI states."
],
"reviewer_note": "Use this profile to judge whether the generated artifacts feel designed for their job, not merely rendered."
}
reports/artifact-design-profile.md
+76
View File
@@ -0,0 +1,76 @@
# Artifact Design Profile
Skill: `yao-meta-skill`
Design system: `content-led editorial`
## Primary Artifact Direction
**Report or brief**
High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
## Matched Artifact Families
### Report or brief
- Matched keywords: report, summary
- Score: `2`
- Direction: High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
### Code, CLI, or implementation guide
- Matched keywords: script
- Score: `1`
- Direction: Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.
## Layout Patterns To Prefer
- thesis
- evidence blocks
- decision table
- risks
- next actions
- prerequisites
- commands
- expected output
## Design Tokens
### Type
- Use a distinctive display face or serif for major claims when the artifact is editorial.
- Use a restrained sans for dense body text and technical details.
- Use mono only for metadata, paths, commands, labels, and evidence tags.
### Color
- Choose colors from the artifact's domain, brand, or evidence mood.
- Do not default to Kami parchment, purple gradients, or generic SaaS blue unless the content justifies it.
- Keep accent color limited to decisions, active states, risk, or section anchors.
### Spacing
- Prefer clear grid rhythm over floating decorative cards.
- Increase whitespace around decisions and shrink whitespace around supporting metadata.
- Split dense content instead of shrinking type or adding scroll traps.
### Components
- Use cards for grouped evidence, tables for comparisons, callouts for decisions, and timelines for sequence.
- Avoid cards inside cards.
- Keep reviewer-only detail visible but visually quieter than user-facing guidance.
## Quality Gates
- Keep the first screen useful without requiring the reader to parse every detail.
- Use tables only for comparisons; move explanations below the table.
- Keep source notes readable without flooding the body with markers.
- Name the working directory and required inputs for commands.
- Mark destructive, networked, or external side-effect operations.
- Prefer the smallest runnable snippet over broad framework scaffolding.
## Anti-Patterns
- Do not copy Kami's fixed parchment background as a default.
- Do not use generic purple gradients, glass cards, or stock SaaS hero sections unless the content calls for them.
- Do not let Markdown tables become the default shape for every comparison or explanation.
- Do not turn reviewer evidence into user-facing clutter.
- Do not invent screenshots, citations, charts, or UI states.
## Reviewer Note
Use this profile to judge whether the generated artifacts feel designed for their job, not merely rendered.
reports/baseline-compare.json
+36 -36
View File
@@ -1,10 +1,10 @@
{
"summary": {
"target_count": 3,
"baseline_total_errors": 10,
"baseline_total_errors": 8,
"current_total_errors": 5,
"winner_total_errors": 5,
"winner_vs_baseline_gain": 5,
"winner_vs_baseline_gain": 3,
"winner_vs_current_gain": 0
},
"comparisons": [
@@ -13,7 +13,7 @@
"winner_label": "Current",
"baseline": {
"tokens": 8,
"dev_errors": 2,
"dev_errors": 1,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
@@ -35,45 +35,45 @@
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"delta": {
"current_vs_baseline": 2,
"winner_vs_baseline": 2,
"winner_vs_current": 0
}
},
{
"label": "team-frontend-review",
"winner_label": "Current",
"baseline": {
"tokens": 52,
"dev_errors": 4,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"current": {
"tokens": 50,
"dev_errors": 3,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"winner": {
"tokens": 50,
"dev_errors": 3,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"delta": {
"current_vs_baseline": 1,
"winner_vs_baseline": 1,
"winner_vs_current": 0
}
},
{
"label": "team-frontend-review",
"winner_label": "Current",
"baseline": {
"tokens": 52,
"dev_errors": 3,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"current": {
"tokens": 50,
"dev_errors": 3,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"winner": {
"tokens": 50,
"dev_errors": 3,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"delta": {
"current_vs_baseline": 0,
"winner_vs_baseline": 0,
"winner_vs_current": 0
}
},
{
"label": "governed-incident-command",
"winner_label": "Current",
reports/baseline-compare.md
+4 -4
View File
@@ -3,16 +3,16 @@
A lightweight with-skill vs baseline comparison across tracked optimization targets.
- Targets: `3`
- Baseline total errors: `10`
- Baseline total errors: `8`
- Current total errors: `5`
- Winner total errors: `5`
- Winner vs baseline gain: `5`
- Winner vs baseline gain: `3`
- Winner vs current gain: `0`
## Target Breakdown
| Target | Baseline Errors | Current Errors | Winner Errors | Winner Label |
| --- | ---: | ---: | ---: | --- |
| root | 2 | 0 | 0 | Current |
| team-frontend-review | 4 | 3 | 3 | Current |
| root | 1 | 0 | 0 | Current |
| team-frontend-review | 3 | 3 | 3 | Current |
| governed-incident-command | 4 | 2 | 2 | Current |
reports/candidate_registry.json
+51 -53
View File
@@ -36,7 +36,7 @@
"calibration": {
"holdout": {
"threshold": 0.33,
"mean_positive_score": 0.672,
"mean_positive_score": 0.673,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.468,
@@ -55,11 +55,11 @@
"mean_non_trigger_score": 0.013,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.026,
"score_gap": 0.366,
"max_non_trigger_score": 0.028,
"score_gap": 0.364,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.304,
"negative_threshold_buffer": 0.302,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
@@ -164,7 +164,7 @@
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.607,
"average_margin": 0.611,
"misroute_count": 0,
"ambiguous_case_count": 0
}
@@ -178,7 +178,7 @@
"promotion_state": "reference",
"description": "Create and improve agent skills.",
"tokens": 8,
"dev_errors": 2,
"dev_errors": 1,
"holdout_errors": 0
},
{
@@ -201,7 +201,21 @@
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 41,
"dev_errors": 3,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 56,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
@@ -213,9 +227,9 @@
"label": "Balanced",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 54,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 60,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
@@ -227,9 +241,9 @@
"label": "Artifact Aware",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 72,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 77,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit",
@@ -242,29 +256,14 @@
"label": "Boundary",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 78,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 83,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit",
"longer_without_gain"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 50,
"dev_errors": 6,
"holdout_errors": 2,
"reason_tags": [
"weaker_dev_fit",
"visible_holdout_regression"
]
}
],
"decision": "keep_current"
@@ -305,14 +304,14 @@
"calibration": {
"holdout": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_non_trigger_score": 0.088,
"mean_near_neighbor_score": 0.042,
"min_positive_score": 0.49,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.089,
"mean_near_neighbor_score": 0.045,
"min_positive_score": 0.496,
"max_non_trigger_score": 0.211,
"score_gap": 0.279,
"threshold_margin": 0.01,
"positive_threshold_buffer": 0.01,
"score_gap": 0.285,
"threshold_margin": 0.016,
"positive_threshold_buffer": 0.016,
"negative_threshold_buffer": 0.269,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -320,7 +319,7 @@
},
"blind_holdout": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.177,
"mean_near_neighbor_score": 0.226,
"min_positive_score": 0.504,
@@ -434,7 +433,7 @@
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.8,
"average_margin": 0.805,
"misroute_count": 0,
"ambiguous_case_count": 0
}
@@ -448,7 +447,7 @@
"promotion_state": "reference",
"description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.",
"tokens": 52,
"dev_errors": 4,
"dev_errors": 3,
"holdout_errors": 0
},
{
@@ -558,29 +557,29 @@
"calibration": {
"holdout": {
"threshold": 0.48,
"mean_positive_score": 0.367,
"mean_positive_score": 0.37,
"mean_non_trigger_score": 0.019,
"mean_near_neighbor_score": 0.038,
"mean_near_neighbor_score": 0.037,
"min_positive_score": 0.08,
"max_non_trigger_score": 0.058,
"score_gap": 0.022,
"max_non_trigger_score": 0.057,
"score_gap": 0.023,
"threshold_margin": -0.4,
"positive_threshold_buffer": -0.4,
"negative_threshold_buffer": 0.422,
"negative_threshold_buffer": 0.423,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "overlap"
},
"blind_holdout": {
"threshold": 0.48,
"mean_positive_score": 0.899,
"mean_positive_score": 0.902,
"mean_non_trigger_score": 0.168,
"mean_near_neighbor_score": 0.138,
"min_positive_score": 0.798,
"min_positive_score": 0.804,
"max_non_trigger_score": 0.386,
"score_gap": 0.412,
"score_gap": 0.418,
"threshold_margin": 0.094,
"positive_threshold_buffer": 0.318,
"positive_threshold_buffer": 0.324,
"negative_threshold_buffer": 0.094,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -694,7 +693,7 @@
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.654,
"average_margin": 0.659,
"misroute_count": 0,
"ambiguous_case_count": 0
}
@@ -747,9 +746,8 @@
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"tokens": 54,
"dev_errors": 1,
"holdout_errors": 2,
"holdout_errors": 1,
"reason_tags": [
"visible_holdout_regression",
"longer_without_gain"
]
},
reports/candidate_registry.md
+8 -8
View File
@@ -2,14 +2,14 @@
| Target | Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors | Reason Tags |
| --- | --- | --- | --- | --- | ---: | ---: | ---: | --- |
| `yao-meta-skill` | baseline | `Baseline` | reference | reference | 8 | 2 | 0 | - |
| `yao-meta-skill` | baseline | `Baseline` | reference | reference | 8 | 1 | 0 | - |
| `yao-meta-skill` | current | `Current` | selected_by_dev | kept_current | 65 | 0 | 0 | - |
| `yao-meta-skill` | candidate | `Minimal` | not_selected | blocked | 41 | 3 | 0 | weaker_dev_fit |
| `yao-meta-skill` | candidate | `Balanced` | not_selected | blocked | 54 | 3 | 0 | weaker_dev_fit |
| `yao-meta-skill` | candidate | `Artifact Aware` | not_selected | blocked | 72 | 3 | 0 | weaker_dev_fit, longer_without_gain |
| `yao-meta-skill` | candidate | `Boundary` | not_selected | blocked | 78 | 3 | 0 | weaker_dev_fit, longer_without_gain |
| `yao-meta-skill` | candidate | `Guardrail` | not_selected | blocked | 50 | 6 | 2 | weaker_dev_fit, visible_holdout_regression |
| `team-frontend-review` | baseline | `Baseline` | reference | reference | 52 | 4 | 0 | - |
| `yao-meta-skill` | candidate | `Minimal` | not_selected | blocked | 41 | 2 | 0 | weaker_dev_fit |
| `yao-meta-skill` | candidate | `Guardrail` | not_selected | blocked | 56 | 2 | 0 | weaker_dev_fit |
| `yao-meta-skill` | candidate | `Balanced` | not_selected | blocked | 60 | 2 | 0 | weaker_dev_fit |
| `yao-meta-skill` | candidate | `Artifact Aware` | not_selected | blocked | 77 | 2 | 0 | weaker_dev_fit, longer_without_gain |
| `yao-meta-skill` | candidate | `Boundary` | not_selected | blocked | 83 | 2 | 0 | weaker_dev_fit, longer_without_gain |
| `team-frontend-review` | baseline | `Baseline` | reference | reference | 52 | 3 | 0 | - |
| `team-frontend-review` | current | `Current` | selected_by_dev | kept_current | 50 | 3 | 0 | - |
| `team-frontend-review` | candidate | `Guardrail` | not_selected | blocked | 62 | 3 | 0 | longer_without_gain |
| `team-frontend-review` | candidate | `Balanced` | not_selected | blocked | 64 | 3 | 0 | longer_without_gain |
@@ -18,6 +18,6 @@
| `governed-incident-command` | baseline | `Baseline` | reference | reference | 93 | 1 | 2 | - |
| `governed-incident-command` | current | `Current` | selected_by_dev | kept_current | 37 | 1 | 1 | - |
| `governed-incident-command` | candidate | `Guardrail` | not_selected | blocked | 51 | 1 | 2 | visible_holdout_regression, longer_without_gain |
| `governed-incident-command` | candidate | `Balanced` | not_selected | blocked | 54 | 1 | 2 | visible_holdout_regression, longer_without_gain |
| `governed-incident-command` | candidate | `Balanced` | not_selected | blocked | 54 | 1 | 1 | longer_without_gain |
| `governed-incident-command` | candidate | `Boundary` | not_selected | blocked | 78 | 1 | 1 | longer_without_gain |
| `governed-incident-command` | candidate | `Artifact Aware` | not_selected | blocked | 78 | 1 | 1 | longer_without_gain |
reports/context_budget.json
+6 -6
View File
@@ -5,13 +5,13 @@
"stats": {
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 818,
"other_text_tokens": 295144,
"estimated_initial_load_tokens": 994,
"estimated_total_text_tokens": 295962,
"relevant_file_count": 184,
"skill_body_tokens": 775,
"other_text_tokens": 321216,
"estimated_initial_load_tokens": 951,
"estimated_total_text_tokens": 321991,
"relevant_file_count": 205,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 130.8
"quality_density": 136.7
}
}
reports/context_budget.md
+1 -1
View File
@@ -2,7 +2,7 @@
| Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status |
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |
| root | `.` | `production` | 1000 | 994 | 818 | 130.8 | - | ok |
| root | `.` | `production` | 1000 | 951 | 775 | 136.7 | - | ok |
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok |
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok |
reports/context_budget_summary.json
+3 -3
View File
@@ -6,9 +6,9 @@
"path": ".",
"budget_tier": "production",
"budget_limit": 1000,
"initial_tokens": 994,
"skill_body_tokens": 818,
"quality_density": 130.8,
"initial_tokens": 951,
"skill_body_tokens": 775,
"quality_density": 136.7,
"unused_resource_dirs": [],
"ok": true,
"warnings": []
reports/description_optimization.json
+228 -206
View File
File diff suppressed because it is too large Load Diff
reports/description_optimization.md
+6 -6
View File
@@ -15,11 +15,11 @@ Create, refactor, evaluate, and package agent skills from workflows, prompts, tr
| Candidate | Tokens | Dev FP | Dev FN | Dev Near | Holdout FP | Holdout FN |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| `Current` | 65 | 0 | 0 | 1.0 | 0 | 0 |
| `Minimal` | 41 | 2 | 1 | 0.857 | 0 | 0 |
| `Balanced` | 54 | 2 | 1 | 0.857 | 0 | 0 |
| `Artifact Aware` | 72 | 2 | 1 | 0.857 | 0 | 0 |
| `Boundary` | 78 | 2 | 1 | 0.857 | 0 | 0 |
| `Guardrail` | 50 | 5 | 1 | 0.429 | 2 | 0 |
| `Minimal` | 41 | 1 | 1 | 1.0 | 0 | 0 |
| `Guardrail` | 56 | 1 | 1 | 1.0 | 0 | 0 |
| `Balanced` | 60 | 1 | 1 | 1.0 | 0 | 0 |
| `Artifact Aware` | 77 | 1 | 1 | 1.0 | 0 | 0 |
| `Boundary` | 83 | 1 | 1 | 1.0 | 0 | 0 |
## Acceptance Gates
@@ -35,7 +35,7 @@ Create, refactor, evaluate, and package agent skills from workflows, prompts, tr
| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |
| --- | ---: | --- | ---: | ---: | ---: |
| Holdout | 0.468 | healthy | 0.0 | 0.468 | 0.416 |
| Blind Holdout | 0.366 | watch | 0.333 | 0.366 | 0.369 |
| Blind Holdout | 0.364 | watch | 0.333 | 0.364 | 0.368 |
| Adversarial Holdout | 0.834 | healthy | 0.0 | 0.834 | 0.845 |
## Judge Blind Summary
reports/description_optimization_suite.json
+21 -21
View File
@@ -36,7 +36,7 @@
"calibration": {
"holdout": {
"threshold": 0.33,
"mean_positive_score": 0.672,
"mean_positive_score": 0.673,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.468,
@@ -55,11 +55,11 @@
"mean_non_trigger_score": 0.013,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.026,
"score_gap": 0.366,
"max_non_trigger_score": 0.028,
"score_gap": 0.364,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.304,
"negative_threshold_buffer": 0.302,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
@@ -178,14 +178,14 @@
"calibration": {
"holdout": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_non_trigger_score": 0.088,
"mean_near_neighbor_score": 0.042,
"min_positive_score": 0.49,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.089,
"mean_near_neighbor_score": 0.045,
"min_positive_score": 0.496,
"max_non_trigger_score": 0.211,
"score_gap": 0.279,
"threshold_margin": 0.01,
"positive_threshold_buffer": 0.01,
"score_gap": 0.285,
"threshold_margin": 0.016,
"positive_threshold_buffer": 0.016,
"negative_threshold_buffer": 0.269,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -193,7 +193,7 @@
},
"blind_holdout": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.177,
"mean_near_neighbor_score": 0.226,
"min_positive_score": 0.504,
@@ -320,29 +320,29 @@
"calibration": {
"holdout": {
"threshold": 0.48,
"mean_positive_score": 0.367,
"mean_positive_score": 0.37,
"mean_non_trigger_score": 0.019,
"mean_near_neighbor_score": 0.038,
"mean_near_neighbor_score": 0.037,
"min_positive_score": 0.08,
"max_non_trigger_score": 0.058,
"score_gap": 0.022,
"max_non_trigger_score": 0.057,
"score_gap": 0.023,
"threshold_margin": -0.4,
"positive_threshold_buffer": -0.4,
"negative_threshold_buffer": 0.422,
"negative_threshold_buffer": 0.423,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "overlap"
},
"blind_holdout": {
"threshold": 0.48,
"mean_positive_score": 0.899,
"mean_positive_score": 0.902,
"mean_non_trigger_score": 0.168,
"mean_near_neighbor_score": 0.138,
"min_positive_score": 0.798,
"min_positive_score": 0.804,
"max_non_trigger_score": 0.386,
"score_gap": 0.412,
"score_gap": 0.418,
"threshold_margin": 0.094,
"positive_threshold_buffer": 0.318,
"positive_threshold_buffer": 0.324,
"negative_threshold_buffer": 0.094,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
reports/eval_suite.json
+77 -65
View File
@@ -351,7 +351,7 @@
{
"prompt": "Turn this runbook into a reusable agent skill.",
"family": "workflow_to_skill",
"score": 0.729,
"score": 0.739,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -375,7 +375,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.625,
"lexical_support": 0.75,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -448,7 +448,7 @@
{
"prompt": "Improve this skill description and add evals.",
"family": "iterate_existing_skill",
"score": 0.353,
"score": 0.365,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -471,7 +471,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.291,
"support_score": 0,
"lexical_support": 0.571,
"lexical_support": 0.714,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -492,7 +492,7 @@
{
"prompt": "Refactor this prompt into a proper skill package.",
"family": "iterate_existing_skill",
"score": 0.532,
"score": 0.542,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -515,7 +515,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.5,
"lexical_support": 0.625,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -534,7 +534,7 @@
{
"prompt": "Package this skill for team reuse.",
"family": "package_for_team",
"score": 0.448,
"score": 0.455,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -558,7 +558,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.667,
"lexical_support": 0.833,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -584,7 +584,7 @@
{
"prompt": "Create a meta-skill for packaging internal workflows.",
"family": "meta_skill_creation",
"score": 0.591,
"score": 0.595,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -608,7 +608,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.527,
"support_score": 0,
"lexical_support": 0.571,
"lexical_support": 0.625,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -632,7 +632,7 @@
{
"prompt": "Standardize this recurring workflow as a shareable skill package with references.",
"family": "package_for_team",
"score": 0.749,
"score": 0.756,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -659,7 +659,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.709,
"support_score": 0.06,
"lexical_support": 0.273,
"lexical_support": 0.364,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -731,7 +731,7 @@
{
"prompt": "Operationalize this workflow as a reusable skill with validation notes and team library metadata.",
"family": "paraphrase_trigger",
"score": 0.88,
"score": 0.886,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -757,7 +757,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0,
"lexical_support": 0.429,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -1062,7 +1062,7 @@
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.25,
"lexical_support": 0.2,
"negative_penalty": 0.53,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1266,7 +1266,7 @@
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.19,
"lexical_support": 0.174,
"negative_penalty": 0.72,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -1316,7 +1316,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.2,
"lexical_support": 0.267,
"negative_penalty": 0.18,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1364,7 +1364,7 @@
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.235,
"lexical_support": 0.294,
"negative_penalty": 0.77,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -1421,7 +1421,7 @@
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.429,
"lexical_support": 0.375,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1565,7 +1565,7 @@
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.4,
"lexical_support": 0.455,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1661,7 +1661,7 @@
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.231,
"lexical_support": 0.357,
"negative_penalty": 0.42,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -1712,7 +1712,7 @@
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.158,
"lexical_support": 0.211,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -1795,14 +1795,14 @@
},
"comparison": {
"baseline_false_positives": 0,
"baseline_false_negatives": 1,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": -1,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 0.917,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"returncode": 0
@@ -1994,7 +1994,7 @@
{
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill",
"score": 0.388,
"score": 0.401,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2018,7 +2018,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.333,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -2101,7 +2101,7 @@
{
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset",
"score": 0.418,
"score": 0.422,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2126,7 +2126,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.357,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2150,7 +2150,7 @@
{
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill",
"score": 0.557,
"score": 0.568,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2175,7 +2175,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.357,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -2201,7 +2201,7 @@
{
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
"family": "paraphrase_trigger",
"score": 0.471,
"score": 0.474,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2225,7 +2225,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0,
"lexical_support": 0.538,
"lexical_support": 0.571,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -2307,7 +2307,7 @@
{
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
"family": "long_context_trigger",
"score": 0.888,
"score": 0.889,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -2335,7 +2335,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0.06,
"lexical_support": 0.333,
"lexical_support": 0.346,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -2543,7 +2543,7 @@
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.273,
"lexical_support": 0.333,
"negative_penalty": 0.35,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -2634,7 +2634,7 @@
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.182,
"lexical_support": 0.217,
"negative_penalty": 0.68,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -2659,7 +2659,7 @@
{
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
"family": "long_context_summary_only",
"score": 0.257,
"score": 0.256,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -2687,7 +2687,7 @@
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.353,
"lexical_support": 0.333,
"negative_penalty": 0.3,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -2798,7 +2798,7 @@
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.286,
"lexical_support": 0.357,
"negative_penalty": 0.42,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -2846,7 +2846,7 @@
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.4,
"lexical_support": 0.6,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2901,7 +2901,7 @@
],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.286,
"lexical_support": 0.267,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2955,7 +2955,7 @@
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.462,
"lexical_support": 0.5,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -2982,7 +2982,7 @@
{
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
"family": "long_context_near_neighbor",
"score": 0.328,
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
@@ -3002,13 +3002,18 @@
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only"
"document_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.18,
"lexical_support": 0.238,
"negative_penalty": 0.82,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
@@ -3025,11 +3030,18 @@
"wiki",
"reference document",
"template only"
],
"future_outline": [
"not now",
"later but not now"
],
"no_build_directive": [
"not now",
"later but not now"
]
}
}
},
"boundary_case": true
}
},
{
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
@@ -3063,7 +3075,7 @@
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.278,
"lexical_support": 0.263,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -3090,13 +3102,13 @@
]
},
"comparison": {
"baseline_false_positives": 1,
"baseline_false_positives": 0,
"baseline_false_negatives": 1,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": -1,
"false_positive_delta": 0,
"false_negative_delta": -1,
"baseline_precision": 0.875,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 0.875,
"improved_recall": 1.0
@@ -3219,7 +3231,7 @@
{
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset",
"score": 0.468,
"score": 0.472,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -3245,7 +3257,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0.06,
"lexical_support": 0.316,
"lexical_support": 0.368,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -3272,7 +3284,7 @@
{
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill",
"score": 0.729,
"score": 0.731,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -3298,7 +3310,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.691,
"support_score": 0,
"lexical_support": 0.412,
"lexical_support": 0.444,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -3329,7 +3341,7 @@
{
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill",
"score": 0.563,
"score": 0.568,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
@@ -3353,7 +3365,7 @@
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.429,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -3518,7 +3530,7 @@
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.308,
"lexical_support": 0.286,
"negative_penalty": 0.72,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -3628,7 +3640,7 @@
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.222,
"lexical_support": 0.278,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
@@ -3683,7 +3695,7 @@
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.25,
"lexical_support": 0.312,
"negative_penalty": 0.77,
"coverage_boost": 0.06,
"concept_evidence": {
@@ -3842,7 +3854,7 @@
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.538,
"lexical_support": 0.615,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
@@ -3899,7 +3911,7 @@
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.375,
"lexical_support": 0.438,
"negative_penalty": 0.6,
"coverage_boost": 0.06,
"concept_evidence": {
reports/iteration_bundles/governed-incident-command/bundle.json
+1 -2
View File
@@ -67,9 +67,8 @@
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"tokens": 54,
"dev_errors": 1,
"holdout_errors": 2,
"holdout_errors": 1,
"reason_tags": [
"visible_holdout_regression",
"longer_without_gain"
]
},
reports/iteration_bundles/governed-incident-command/bundle.md
+1 -1
View File
@@ -31,7 +31,7 @@
| baseline | `Baseline` | reference | reference | 93 | 1 | 2 |
| current | `Current` | selected_by_dev | kept_current | 37 | 1 | 1 |
| candidate | `Guardrail` | not_selected | blocked | 51 | 1 | 2 |
| candidate | `Balanced` | not_selected | blocked | 54 | 1 | 2 |
| candidate | `Balanced` | not_selected | blocked | 54 | 1 | 1 |
| candidate | `Boundary` | not_selected | blocked | 78 | 1 | 1 |
| candidate | `Artifact Aware` | not_selected | blocked | 78 | 1 | 1 |
reports/iteration_bundles/team-frontend-review/bundle.json
+1 -1
View File
@@ -27,7 +27,7 @@
"promotion_state": "reference",
"description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.",
"tokens": 52,
"dev_errors": 4,
"dev_errors": 3,
"holdout_errors": 0
},
{
reports/iteration_bundles/team-frontend-review/bundle.md
+1 -1
View File
@@ -27,7 +27,7 @@
| Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors |
| --- | --- | --- | --- | ---: | ---: | ---: |
| baseline | `Baseline` | reference | reference | 52 | 4 | 0 |
| baseline | `Baseline` | reference | reference | 52 | 3 | 0 |
| current | `Current` | selected_by_dev | kept_current | 50 | 3 | 0 |
| candidate | `Guardrail` | not_selected | blocked | 62 | 3 | 0 |
| candidate | `Balanced` | not_selected | blocked | 64 | 3 | 0 |
reports/iteration_bundles/yao-meta-skill/bundle.json
+25 -26
View File
@@ -26,7 +26,7 @@
"promotion_state": "reference",
"description": "Create and improve agent skills.",
"tokens": 8,
"dev_errors": 2,
"dev_errors": 1,
"holdout_errors": 0
},
{
@@ -49,7 +49,21 @@
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 41,
"dev_errors": 3,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 56,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
@@ -61,9 +75,9 @@
"label": "Balanced",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 54,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 60,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
@@ -75,9 +89,9 @@
"label": "Artifact Aware",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 72,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 77,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit",
@@ -90,29 +104,14 @@
"label": "Boundary",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 78,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 83,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit",
"longer_without_gain"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 50,
"dev_errors": 6,
"holdout_errors": 2,
"reason_tags": [
"weaker_dev_fit",
"visible_holdout_regression"
]
}
],
"review_template": {
reports/iteration_bundles/yao-meta-skill/bundle.md
+6 -6
View File
@@ -26,13 +26,13 @@
| Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors |
| --- | --- | --- | --- | ---: | ---: | ---: |
| baseline | `Baseline` | reference | reference | 8 | 2 | 0 |
| baseline | `Baseline` | reference | reference | 8 | 1 | 0 |
| current | `Current` | selected_by_dev | kept_current | 65 | 0 | 0 |
| candidate | `Minimal` | not_selected | blocked | 41 | 3 | 0 |
| candidate | `Balanced` | not_selected | blocked | 54 | 3 | 0 |
| candidate | `Artifact Aware` | not_selected | blocked | 72 | 3 | 0 |
| candidate | `Boundary` | not_selected | blocked | 78 | 3 | 0 |
| candidate | `Guardrail` | not_selected | blocked | 50 | 6 | 2 |
| candidate | `Minimal` | not_selected | blocked | 41 | 2 | 0 |
| candidate | `Guardrail` | not_selected | blocked | 56 | 2 | 0 |
| candidate | `Balanced` | not_selected | blocked | 60 | 2 | 0 |
| candidate | `Artifact Aware` | not_selected | blocked | 77 | 2 | 0 |
| candidate | `Boundary` | not_selected | blocked | 83 | 2 | 0 |
## Human Review Stub
reports/iteration_ledger.md
+3 -3
View File
@@ -43,7 +43,7 @@
| Route | Precision | Recall | Avg Margin |
| --- | ---: | ---: | ---: |
| `yao-meta-skill` | 1.0 | 1.0 | 0.607 |
| `team-frontend-review` | 1.0 | 1.0 | 0.8 |
| `governed-incident-command` | 1.0 | 1.0 | 0.654 |
| `yao-meta-skill` | 1.0 | 1.0 | 0.611 |
| `team-frontend-review` | 1.0 | 1.0 | 0.805 |
| `governed-incident-command` | 1.0 | 1.0 | 0.659 |
| `no_route` | 1.0 | 1.0 | - |
reports/promotion_decisions.json
+51 -53
View File
@@ -42,7 +42,7 @@
"calibration": {
"holdout": {
"threshold": 0.33,
"mean_positive_score": 0.672,
"mean_positive_score": 0.673,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.468,
@@ -61,11 +61,11 @@
"mean_non_trigger_score": 0.013,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.026,
"score_gap": 0.366,
"max_non_trigger_score": 0.028,
"score_gap": 0.364,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.304,
"negative_threshold_buffer": 0.302,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
@@ -170,7 +170,7 @@
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.607,
"average_margin": 0.611,
"misroute_count": 0,
"ambiguous_case_count": 0
}
@@ -184,7 +184,7 @@
"promotion_state": "reference",
"description": "Create and improve agent skills.",
"tokens": 8,
"dev_errors": 2,
"dev_errors": 1,
"holdout_errors": 0
},
{
@@ -207,7 +207,21 @@
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 41,
"dev_errors": 3,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 56,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
@@ -219,9 +233,9 @@
"label": "Balanced",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 54,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 60,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
@@ -233,9 +247,9 @@
"label": "Artifact Aware",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 72,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 77,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit",
@@ -248,29 +262,14 @@
"label": "Boundary",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 78,
"dev_errors": 3,
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 83,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit",
"longer_without_gain"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 50,
"dev_errors": 6,
"holdout_errors": 2,
"reason_tags": [
"weaker_dev_fit",
"visible_holdout_regression"
]
}
],
"decision": "keep_current"
@@ -311,14 +310,14 @@
"calibration": {
"holdout": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_non_trigger_score": 0.088,
"mean_near_neighbor_score": 0.042,
"min_positive_score": 0.49,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.089,
"mean_near_neighbor_score": 0.045,
"min_positive_score": 0.496,
"max_non_trigger_score": 0.211,
"score_gap": 0.279,
"threshold_margin": 0.01,
"positive_threshold_buffer": 0.01,
"score_gap": 0.285,
"threshold_margin": 0.016,
"positive_threshold_buffer": 0.016,
"negative_threshold_buffer": 0.269,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -326,7 +325,7 @@
},
"blind_holdout": {
"threshold": 0.48,
"mean_positive_score": 0.681,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.177,
"mean_near_neighbor_score": 0.226,
"min_positive_score": 0.504,
@@ -440,7 +439,7 @@
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.8,
"average_margin": 0.805,
"misroute_count": 0,
"ambiguous_case_count": 0
}
@@ -454,7 +453,7 @@
"promotion_state": "reference",
"description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.",
"tokens": 52,
"dev_errors": 4,
"dev_errors": 3,
"holdout_errors": 0
},
{
@@ -564,29 +563,29 @@
"calibration": {
"holdout": {
"threshold": 0.48,
"mean_positive_score": 0.367,
"mean_positive_score": 0.37,
"mean_non_trigger_score": 0.019,
"mean_near_neighbor_score": 0.038,
"mean_near_neighbor_score": 0.037,
"min_positive_score": 0.08,
"max_non_trigger_score": 0.058,
"score_gap": 0.022,
"max_non_trigger_score": 0.057,
"score_gap": 0.023,
"threshold_margin": -0.4,
"positive_threshold_buffer": -0.4,
"negative_threshold_buffer": 0.422,
"negative_threshold_buffer": 0.423,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "overlap"
},
"blind_holdout": {
"threshold": 0.48,
"mean_positive_score": 0.899,
"mean_positive_score": 0.902,
"mean_non_trigger_score": 0.168,
"mean_near_neighbor_score": 0.138,
"min_positive_score": 0.798,
"min_positive_score": 0.804,
"max_non_trigger_score": 0.386,
"score_gap": 0.412,
"score_gap": 0.418,
"threshold_margin": 0.094,
"positive_threshold_buffer": 0.318,
"positive_threshold_buffer": 0.324,
"negative_threshold_buffer": 0.094,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
@@ -700,7 +699,7 @@
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.654,
"average_margin": 0.659,
"misroute_count": 0,
"ambiguous_case_count": 0
}
@@ -753,9 +752,8 @@
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"tokens": 54,
"dev_errors": 1,
"holdout_errors": 2,
"holdout_errors": 1,
"reason_tags": [
"visible_holdout_regression",
"longer_without_gain"
]
},
reports/reference-synthesis.json
+71
View File
@@ -59,6 +59,77 @@
"Use research tracks to justify the smallest evaluation loop that still catches regressions.",
"Use principle tracks to keep the package small, boundary-aware, and outcome-driven."
],
"pattern_gate": {
"threshold": 4,
"source_count": 3,
"accepted": [
{
"name": "Official workflow product ergonomics",
"source_type": "official",
"borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.",
"avoid": "Do not mimic product polish that adds UI bulk without improving clarity.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"distinctiveness",
"boundary"
],
"missing": [],
"score": 4
}
],
"deferred": [
{
"name": "Hypothesis-test-learn loop",
"source_type": "research",
"borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.",
"avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": false,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"boundary"
],
"missing": [
"distinctiveness"
],
"score": 3
},
{
"name": "Boundary-first design",
"source_type": "principles",
"borrow": "Borrow the discipline of defining what the skill should not own before growing the package.",
"avoid": "Do not expand execution assets until route boundaries stay clean.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": false,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"boundary"
],
"missing": [
"distinctiveness"
],
"score": 3
}
],
"summary": "1 accepted, 2 deferred using threshold 4/4."
},
"conflicts": [],
"recommendation": {
"summary": "Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity.",
reports/reference-synthesis.md
+10
View File
@@ -43,6 +43,16 @@ Skill: `yao-meta-skill`
- Do not create experimental overhead that exceeds the skill's real risk tier.
- Do not expand execution assets until route boundaries stay clean.
## Pattern Gate
- Summary: 1 accepted, 2 deferred using threshold 4/4.
- Acceptance threshold: `4/4`
- Accepted patterns:
- **Official workflow product ergonomics**: 4/4 (recurrence, generativity, distinctiveness, boundary)
- Deferred patterns:
- **Hypothesis-test-learn loop**: missing distinctiveness
- **Boundary-first design**: missing distinctiveness
## Default Recommendation
- Summary: Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity.
reports/route_scorecard.json
+22 -22
View File
@@ -15,7 +15,7 @@
"correct_count": 3,
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.607
"average_margin": 0.611
},
"team-frontend-review": {
"expected_count": 3,
@@ -23,7 +23,7 @@
"correct_count": 3,
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.8
"average_margin": 0.805
},
"governed-incident-command": {
"expected_count": 3,
@@ -31,7 +31,7 @@
"correct_count": 3,
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.654
"average_margin": 0.659
},
"no_route": {
"expected_count": 4,
@@ -118,12 +118,12 @@
"expected_route": "yao-meta-skill",
"predicted_route": "yao-meta-skill",
"correct": true,
"margin": 0.529,
"margin": 0.535,
"ambiguous": false,
"ranked_routes": [
{
"name": "yao-meta-skill",
"score": 0.529,
"score": 0.535,
"passed_threshold": true,
"threshold": 0.33,
"matched_desired_concepts": [
@@ -157,12 +157,12 @@
"expected_route": "yao-meta-skill",
"predicted_route": "yao-meta-skill",
"correct": true,
"margin": 0.552,
"margin": 0.558,
"ambiguous": false,
"ranked_routes": [
{
"name": "yao-meta-skill",
"score": 0.552,
"score": 0.558,
"passed_threshold": true,
"threshold": 0.33,
"matched_desired_concepts": [
@@ -197,12 +197,12 @@
"expected_route": "team-frontend-review",
"predicted_route": "team-frontend-review",
"correct": true,
"margin": 0.886,
"margin": 0.893,
"ambiguous": false,
"ranked_routes": [
{
"name": "team-frontend-review",
"score": 0.886,
"score": 0.893,
"passed_threshold": true,
"threshold": 0.48,
"matched_desired_concepts": [
@@ -216,7 +216,7 @@
},
{
"name": "yao-meta-skill",
"score": 0.02,
"score": 0.018,
"passed_threshold": false,
"threshold": 0.33,
"matched_desired_concepts": [],
@@ -224,7 +224,7 @@
},
{
"name": "governed-incident-command",
"score": 0.007,
"score": 0.012,
"passed_threshold": false,
"threshold": 0.48,
"matched_desired_concepts": [],
@@ -238,12 +238,12 @@
"expected_route": "team-frontend-review",
"predicted_route": "team-frontend-review",
"correct": true,
"margin": 0.696,
"margin": 0.704,
"ambiguous": false,
"ranked_routes": [
{
"name": "team-frontend-review",
"score": 0.696,
"score": 0.704,
"passed_threshold": true,
"threshold": 0.48,
"matched_desired_concepts": [
@@ -256,7 +256,7 @@
},
{
"name": "yao-meta-skill",
"score": 0.014,
"score": 0.013,
"passed_threshold": false,
"threshold": 0.33,
"matched_desired_concepts": [],
@@ -264,7 +264,7 @@
},
{
"name": "governed-incident-command",
"score": 0.005,
"score": 0.008,
"passed_threshold": false,
"threshold": 0.48,
"matched_desired_concepts": [],
@@ -296,7 +296,7 @@
},
{
"name": "yao-meta-skill",
"score": 0.013,
"score": 0.012,
"passed_threshold": false,
"threshold": 0.33,
"matched_desired_concepts": [],
@@ -304,7 +304,7 @@
},
{
"name": "governed-incident-command",
"score": 0.0,
"score": 0.006,
"passed_threshold": false,
"threshold": 0.48,
"matched_desired_concepts": [],
@@ -357,12 +357,12 @@
"expected_route": "governed-incident-command",
"predicted_route": "governed-incident-command",
"correct": true,
"margin": 0.797,
"margin": 0.805,
"ambiguous": false,
"ranked_routes": [
{
"name": "governed-incident-command",
"score": 0.797,
"score": 0.805,
"passed_threshold": true,
"threshold": 0.48,
"matched_desired_concepts": [
@@ -383,7 +383,7 @@
},
{
"name": "team-frontend-review",
"score": 0.015,
"score": 0.022,
"passed_threshold": false,
"threshold": 0.48,
"matched_desired_concepts": [],
@@ -397,12 +397,12 @@
"expected_route": "governed-incident-command",
"predicted_route": "governed-incident-command",
"correct": true,
"margin": 0.166,
"margin": 0.172,
"ambiguous": false,
"ranked_routes": [
{
"name": "governed-incident-command",
"score": 0.614,
"score": 0.62,
"passed_threshold": true,
"threshold": 0.48,
"matched_desired_concepts": [
reports/route_scorecard.md
+3 -3
View File
@@ -9,9 +9,9 @@
| Route | Expected | Predicted | Precision | Recall | Avg Margin |
| --- | ---: | ---: | ---: | ---: | ---: |
| `yao-meta-skill` | 3 | 3 | 1.0 | 1.0 | 0.607 |
| `team-frontend-review` | 3 | 3 | 1.0 | 1.0 | 0.8 |
| `governed-incident-command` | 3 | 3 | 1.0 | 1.0 | 0.654 |
| `yao-meta-skill` | 3 | 3 | 1.0 | 1.0 | 0.611 |
| `team-frontend-review` | 3 | 3 | 1.0 | 1.0 | 0.805 |
| `governed-incident-command` | 3 | 3 | 1.0 | 1.0 | 0.659 |
| `no_route` | 4 | 4 | 1.0 | 1.0 | - |
## Confusion Matrix
scripts/init_skill.py
+9 -1
View File
@@ -8,6 +8,7 @@ from github_benchmark_scan import run_github_benchmark_scan
from render_intent_confidence import render_intent_confidence
from render_intent_dialogue import render_intent_dialogue
from render_iteration_directions import render_iteration_directions
from render_artifact_design_profile import render_artifact_design_profile
from render_output_risk_profile import render_output_risk_profile
from render_reference_scan import parse_reference, render_reference_scan
from render_reference_synthesis import render_reference_synthesis
@@ -31,7 +32,9 @@ description: {description}
## Output Quality Guardrails
- Before final output, apply the likely failure modes in `reports/output-risk-profile.md` when that report is present.
- Before rendering reports, tutorials, review pages, dashboards, or visual artifacts, apply the artifact direction and visual quality gates in `reports/artifact-design-profile.md` when that report is present.
- Repair generic headings, cluttered notes, fragile visual assumptions, weak tables, and missing verification cues before handing work back.
- Let the artifact's content choose the visual system; do not copy a fixed palette or report style from another skill without a clear reason.
- If output-specific evidence is missing, state the gap instead of inventing screenshots, citations, data, or examples.
## Honest Boundaries
@@ -61,7 +64,8 @@ README_TEMPLATE = """# {title}
7. Check `reports/skill-overview.html` if you want a fast visual explanation of the package.
8. Open `reports/review-viewer.html` for a compact visual review of the package.
9. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks.
10. Review `reports/iteration-directions.md` for the three most valuable next moves.
10. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns.
11. Review `reports/iteration-directions.md` for the three most valuable next moves.
## Honest Boundaries
@@ -80,6 +84,7 @@ README_TEMPLATE = """# {title}
- `reports/reference-scan.md`: benchmark notes from public references, user references, and local constraints
- `reports/reference-synthesis.md`: a combined view of GitHub benchmarks plus curated world-class pattern tracks
- `reports/output-risk-profile.md`: predicted output failure modes and self-repair constraints for this skill
- `reports/artifact-design-profile.md`: artifact-specific design direction, layout patterns, visual quality gates, and anti-patterns
- `reports/skill-overview.html`: visual overview report
- `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps
- `reports/iteration-directions.md`: the top three next iteration directions
@@ -246,6 +251,7 @@ def initialize_skill(
)
reference_synthesis = render_reference_synthesis(root)
output_risk_profile = render_output_risk_profile(root)
artifact_design_profile = render_artifact_design_profile(root)
overview = render_skill_overview(root)
iteration_directions = render_iteration_directions(root)
review_viewer = render_review_viewer(root)
@@ -265,6 +271,8 @@ def initialize_skill(
"reference_synthesis_json": reference_synthesis["artifacts"]["json"],
"output_risk_profile_md": output_risk_profile["artifacts"]["markdown"],
"output_risk_profile_json": output_risk_profile["artifacts"]["json"],
"artifact_design_profile_md": artifact_design_profile["artifacts"]["markdown"],
"artifact_design_profile_json": artifact_design_profile["artifacts"]["json"],
"iteration_directions_md": iteration_directions["artifacts"]["markdown"],
"iteration_directions_json": iteration_directions["artifacts"]["json"],
"review_viewer_html": review_viewer["artifacts"]["html"],
scripts/render_artifact_design_profile.py
+346
View File
@@ -0,0 +1,346 @@
#!/usr/bin/env python3
import argparse
import json
import re
from pathlib import Path
try:
import yaml
except ImportError: # pragma: no cover
yaml = None
ARTIFACT_FAMILIES = [
{
"key": "tutorial",
"label": "Tutorial or guide",
"keywords": ["tutorial", "guide", "lesson", "course", "walkthrough", "how to", "教程", "指南", "课程"],
"direction": "Progressive instructional layout with domain-specific section names, short success checks, and examples close to the user's real input.",
"layout": ["opening promise", "task-specific sections", "worked example", "success checks", "common mistakes"],
"quality_gates": [
"Replace generic headings with learner- and domain-specific headings.",
"Pair every major step with a visible success check.",
"Do not add screenshots unless they are real, current, and action-relevant.",
],
},
{
"key": "report",
"label": "Report or brief",
"keywords": ["report", "brief", "analysis", "summary", "memo", "white paper", "报告", "简报", "分析", "总结"],
"direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.",
"layout": ["thesis", "evidence blocks", "decision table", "risks", "next actions"],
"quality_gates": [
"Keep the first screen useful without requiring the reader to parse every detail.",
"Use tables only for comparisons; move explanations below the table.",
"Keep source notes readable without flooding the body with markers.",
],
},
{
"key": "review_viewer",
"label": "Review viewer",
"keywords": ["review", "viewer", "compare", "diff", "audit", "评审", "对比", "审查"],
"direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix.",
"layout": ["summary", "variant comparison", "evidence", "risks", "review decision"],
"quality_gates": [
"Make differences visible instead of hiding them in prose.",
"Separate author-facing recommendations from reviewer-only evidence.",
"Surface conflicts clearly and keep routine benchmark synthesis quiet.",
],
},
{
"key": "dashboard",
"label": "Dashboard or metrics page",
"keywords": ["dashboard", "metric", "score", "kpi", "chart", "table", "scorecard", "仪表盘", "指标", "评分", "图表", "表格"],
"direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.",
"layout": ["metric board", "ranked signals", "comparison rows", "interpretation", "action queue"],
"quality_gates": [
"Avoid paragraph-heavy table cells.",
"Keep charts tied to one analytical question each.",
"Preserve stable color meaning across metrics and entities.",
],
},
{
"key": "visual_capture",
"label": "Screenshot or visual evidence",
"keywords": ["screenshot", "screen capture", "image evidence", "figma", "截图", "图片", "截屏", "视觉证据"],
"direction": "Evidence-led visual artifact that records source, viewport, crop intent, and the exact region the reader should inspect.",
"layout": ["source context", "visual frame", "inspection notes", "limitations", "next capture"],
"quality_gates": [
"Never invent missing screenshots or visual states.",
"Record source, viewport, and crop intent.",
"Describe the action-relevant region instead of only saying an image exists.",
],
},
{
"key": "slide_like",
"label": "Slide-like narrative",
"keywords": ["slide", "ppt", "deck", "presentation", "one-pager", "演示", "幻灯片", "一页纸"],
"direction": "Narrative page rhythm with strong section roles, controlled density, and a visual system chosen from the topic rather than a default template.",
"layout": ["hero claim", "supporting proof", "structured contrast", "implication", "closing action"],
"quality_gates": [
"Plan role, density, and rhythm before writing HTML.",
"Avoid three or more repeated surfaces in a row.",
"Use structure, spacing, and type before decoration.",
],
},
{
"key": "code_or_cli",
"label": "Code, CLI, or implementation guide",
"keywords": ["code", "cli", "script", "api", "command", "terminal", "代码", "脚本", "命令", "接口"],
"direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.",
"layout": ["prerequisites", "commands", "expected output", "failure handling", "rollback or cleanup"],
"quality_gates": [
"Name the working directory and required inputs for commands.",
"Mark destructive, networked, or external side-effect operations.",
"Prefer the smallest runnable snippet over broad framework scaffolding.",
],
},
]
DESIGN_TOKENS = {
"type": [
"Use a distinctive display face or serif for major claims when the artifact is editorial.",
"Use a restrained sans for dense body text and technical details.",
"Use mono only for metadata, paths, commands, labels, and evidence tags.",
],
"color": [
"Choose colors from the artifact's domain, brand, or evidence mood.",
"Do not default to Kami parchment, purple gradients, or generic SaaS blue unless the content justifies it.",
"Keep accent color limited to decisions, active states, risk, or section anchors.",
],
"spacing": [
"Prefer clear grid rhythm over floating decorative cards.",
"Increase whitespace around decisions and shrink whitespace around supporting metadata.",
"Split dense content instead of shrinking type or adding scroll traps.",
],
"components": [
"Use cards for grouped evidence, tables for comparisons, callouts for decisions, and timelines for sequence.",
"Avoid cards inside cards.",
"Keep reviewer-only detail visible but visually quieter than user-facing guidance.",
],
}
def parse_frontmatter(text: str) -> tuple[dict, str]:
lines = text.splitlines()
if not lines or lines[0].strip() != "---":
return {}, text
try:
end_index = lines[1:].index("---") + 1
except ValueError:
return {}, text
frontmatter_text = "\n".join(lines[1:end_index])
body = "\n".join(lines[end_index + 1 :]).lstrip()
if yaml is not None:
payload = yaml.safe_load(frontmatter_text) or {}
return payload if isinstance(payload, dict) else {}, body
data = {}
for line in frontmatter_text.splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
data[key.strip()] = value.strip().strip('"')
return data, body
def load_json(path: Path) -> dict:
if not path.exists():
return {}
return json.loads(path.read_text(encoding="utf-8"))
def normalized_text(skill_dir: Path) -> str:
skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8")
frontmatter, body = parse_frontmatter(skill_text)
intent = load_json(skill_dir / "reports" / "intent-confidence.json")
output_risk = load_json(skill_dir / "reports" / "output-risk-profile.json")
context = intent.get("context", {}) if isinstance(intent, dict) else {}
risk_labels = " ".join(item.get("label", "") for item in output_risk.get("risk_families", []))
parts = [
skill_dir.name,
str(frontmatter.get("name", "")),
str(frontmatter.get("description", "")),
body,
str(context.get("job", "")),
str(context.get("primary_output", "")),
" ".join(context.get("real_inputs", []) or []),
" ".join(context.get("constraints", []) or []),
" ".join(context.get("standards", []) or []),
risk_labels,
]
return " ".join(parts).lower()
def match_family(text: str, family: dict) -> tuple[int, list[str]]:
hits = []
for keyword in family["keywords"]:
if keyword.lower() in text:
hits.append(keyword)
return len(hits), hits
def dedupe(items: list[str], limit: int) -> list[str]:
seen = set()
output = []
for item in items:
if not item or item in seen:
continue
seen.add(item)
output.append(item)
if len(output) == limit:
break
return output
def design_system_name(matched: list[dict]) -> str:
keys = {item["key"] for item in matched}
if "dashboard" in keys:
return "metric editorial"
if "review_viewer" in keys:
return "review studio"
if "slide_like" in keys:
return "narrative visual brief"
if "tutorial" in keys:
return "guided learning document"
if "visual_capture" in keys:
return "evidence frame"
return "content-led editorial"
def build_summary(skill_dir: Path) -> dict:
text = normalized_text(skill_dir)
matched = []
for family in ARTIFACT_FAMILIES:
score, hits = match_family(text, family)
if score:
matched.append({**family, "score": score, "matched_keywords": hits})
if not matched:
fallback = next(item for item in ARTIFACT_FAMILIES if item["key"] == "report")
matched = [{**fallback, "score": 0, "matched_keywords": ["general-artifact"]}]
matched = sorted(matched, key=lambda item: item["score"], reverse=True)[:5]
primary = matched[0]
layout_patterns = []
gates = []
for item in matched:
layout_patterns.extend(item["layout"])
gates.extend(item["quality_gates"])
anti_patterns = [
"Do not copy Kami's fixed parchment background as a default.",
"Do not use generic purple gradients, glass cards, or stock SaaS hero sections unless the content calls for them.",
"Do not let Markdown tables become the default shape for every comparison or explanation.",
"Do not turn reviewer evidence into user-facing clutter.",
"Do not invent screenshots, citations, charts, or UI states.",
]
return {
"skill_name": skill_dir.name,
"design_system": design_system_name(matched),
"primary_artifact": {
"key": primary["key"],
"label": primary["label"],
"direction": primary["direction"],
"matched_keywords": primary["matched_keywords"],
},
"artifact_families": [
{
"key": item["key"],
"label": item["label"],
"score": item["score"],
"matched_keywords": item["matched_keywords"],
"direction": item["direction"],
}
for item in matched
],
"layout_patterns": dedupe(layout_patterns, 8),
"design_tokens": DESIGN_TOKENS,
"quality_gates": dedupe(gates, 8),
"anti_patterns": anti_patterns,
"reviewer_note": "Use this profile to judge whether the generated artifacts feel designed for their job, not merely rendered.",
}
def render_markdown(summary: dict) -> str:
lines = [
"# Artifact Design Profile",
"",
f"Skill: `{summary['skill_name']}`",
f"Design system: `{summary['design_system']}`",
"",
"## Primary Artifact Direction",
"",
f"**{summary['primary_artifact']['label']}**",
"",
summary["primary_artifact"]["direction"],
"",
"## Matched Artifact Families",
"",
]
for family in summary["artifact_families"]:
lines.extend(
[
f"### {family['label']}",
f"- Matched keywords: {', '.join(family['matched_keywords'])}",
f"- Score: `{family['score']}`",
f"- Direction: {family['direction']}",
"",
]
)
lines.extend(["## Layout Patterns To Prefer", ""])
for item in summary["layout_patterns"]:
lines.append(f"- {item}")
lines.extend(["", "## Design Tokens", ""])
for key, values in summary["design_tokens"].items():
lines.append(f"### {key.title()}")
for value in values:
lines.append(f"- {value}")
lines.append("")
lines.extend(["## Quality Gates", ""])
for item in summary["quality_gates"]:
lines.append(f"- {item}")
lines.extend(["", "## Anti-Patterns", ""])
for item in summary["anti_patterns"]:
lines.append(f"- {item}")
lines.extend(["", "## Reviewer Note", "", summary["reviewer_note"], ""])
return "\n".join(lines).strip() + "\n"
def render_artifact_design_profile(
skill_dir: Path,
output_md: Path | None = None,
output_json: Path | None = None,
) -> dict:
skill_dir = skill_dir.resolve()
reports_dir = skill_dir / "reports"
reports_dir.mkdir(parents=True, exist_ok=True)
output_md = output_md or reports_dir / "artifact-design-profile.md"
output_json = output_json or reports_dir / "artifact-design-profile.json"
summary = build_summary(skill_dir)
output_md.write_text(render_markdown(summary), encoding="utf-8")
output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
return {
"ok": True,
"skill_dir": str(skill_dir),
"artifacts": {
"markdown": str(output_md),
"json": str(output_json),
},
"summary": summary,
}
def main() -> None:
parser = argparse.ArgumentParser(description="Render artifact design direction and visual quality gates for a skill package.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--output-md")
parser.add_argument("--output-json")
args = parser.parse_args()
result = render_artifact_design_profile(
Path(args.skill_dir),
Path(args.output_md).resolve() if args.output_md else None,
Path(args.output_json).resolve() if args.output_json else None,
)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
scripts/render_output_risk_profile.py
+1 -1
View File
@@ -203,7 +203,7 @@ def build_summary(skill_dir: Path) -> dict:
"matched_keywords": ["general-output-risk"],
},
]
matched = sorted(matched, key=lambda item: item["score"], reverse=True)[:4]
matched = sorted(matched, key=lambda item: item["score"], reverse=True)[:5]
top_risks = []
constraints = []
self_repair = []
scripts/render_review_viewer.py
+45
View File
@@ -8,6 +8,7 @@ from pathlib import Path
from render_intent_confidence import render_intent_confidence
from render_intent_dialogue import render_intent_dialogue
from render_iteration_directions import render_iteration_directions
from render_artifact_design_profile import render_artifact_design_profile
from render_output_risk_profile import render_output_risk_profile
from render_reference_scan import render_reference_scan
from render_reference_synthesis import render_reference_synthesis
@@ -62,6 +63,11 @@ def load_output_risk_summary(skill_dir: Path) -> dict:
return payload if isinstance(payload, dict) else {}
def load_artifact_design_summary(skill_dir: Path) -> dict:
payload = load_json(skill_dir / "reports" / "artifact-design-profile.json")
return payload if isinstance(payload, dict) else {}
def ensure_report_inputs(skill_dir: Path) -> dict:
overview_json = skill_dir / "reports" / "skill-overview.json"
intent_confidence_json = skill_dir / "reports" / "intent-confidence.json"
@@ -69,6 +75,7 @@ def ensure_report_inputs(skill_dir: Path) -> dict:
reference_json = skill_dir / "reports" / "reference-scan.json"
reference_synthesis_json = skill_dir / "reports" / "reference-synthesis.json"
output_risk_json = skill_dir / "reports" / "output-risk-profile.json"
artifact_design_json = skill_dir / "reports" / "artifact-design-profile.json"
directions_json = skill_dir / "reports" / "iteration-directions.json"
overview_payload = load_json(overview_json) if overview_json.exists() else {}
@@ -77,6 +84,7 @@ def ensure_report_inputs(skill_dir: Path) -> dict:
reference_payload = load_json(reference_json) if reference_json.exists() else {}
reference_synthesis_payload = load_json(reference_synthesis_json) if reference_synthesis_json.exists() else {}
output_risk_payload = load_json(output_risk_json) if output_risk_json.exists() else {}
artifact_design_payload = load_json(artifact_design_json) if artifact_design_json.exists() else {}
directions_payload = load_json(directions_json) if directions_json.exists() else {}
intent_confidence = intent_confidence_payload or render_intent_confidence(skill_dir)["summary"]
@@ -84,6 +92,7 @@ def ensure_report_inputs(skill_dir: Path) -> dict:
reference = reference_payload or render_reference_scan(skill_dir, [])["summary"]
reference_synthesis = reference_synthesis_payload or render_reference_synthesis(skill_dir)["summary"]
output_risk = output_risk_payload or render_output_risk_profile(skill_dir)["summary"]
artifact_design = artifact_design_payload or render_artifact_design_profile(skill_dir)["summary"]
overview = overview_payload or render_skill_overview(skill_dir)["summary"]
iteration = directions_payload.get("summary", {}) or render_iteration_directions(skill_dir)["summary"]
feedback = load_feedback_summary(skill_dir)
@@ -93,6 +102,7 @@ def ensure_report_inputs(skill_dir: Path) -> dict:
benchmark = load_benchmark_summary(skill_dir)
reference_synthesis = load_reference_synthesis_summary(skill_dir)
output_risk = load_output_risk_summary(skill_dir) or output_risk
artifact_design = load_artifact_design_summary(skill_dir) or artifact_design
return {
"overview": overview,
"intent_confidence": intent_confidence,
@@ -106,6 +116,7 @@ def ensure_report_inputs(skill_dir: Path) -> dict:
"benchmark": benchmark,
"reference_synthesis": reference_synthesis,
"output_risk": output_risk,
"artifact_design": artifact_design,
}
@@ -238,6 +249,7 @@ def evidence_readiness(report: dict) -> dict:
intent_confidence = report.get("intent_confidence", {})
reference_synthesis = report.get("reference_synthesis", {})
output_risk = report.get("output_risk", {})
artifact_design = report.get("artifact_design", {})
benchmark = report.get("benchmark", {})
synthesis = reference_synthesis.get("synthesis", {}) if isinstance(reference_synthesis, dict) else {}
pattern_gate = synthesis.get("pattern_gate", {}) if isinstance(synthesis, dict) else {}
@@ -269,6 +281,11 @@ def evidence_readiness(report: dict) -> dict:
"status": "ready" if output_risk.get("risk_families") else "needs review",
"detail": f"{len(output_risk.get('risk_families', []))} output risk families attached.",
},
{
"label": "Artifact design profile",
"status": "ready" if artifact_design.get("primary_artifact") else "needs review",
"detail": artifact_design.get("primary_artifact", {}).get("direction", "No artifact design profile attached."),
},
]
ready_count = sum(1 for item in checks if item["status"] == "ready")
return {
@@ -292,6 +309,7 @@ def render_html(report: dict) -> str:
benchmark = report.get("benchmark", {})
reference_synthesis = report.get("reference_synthesis", {})
output_risk = report.get("output_risk", {})
artifact_design = report.get("artifact_design", {})
architecture = architecture_steps(overview)
compare_table_rows = compare_rows(compare)
benchmark_rows = benchmark_cards(benchmark)
@@ -329,6 +347,21 @@ def render_html(report: dict) -> str:
if not output_risk_items:
output_risk_items = "<li>No output risk profile attached yet. Generate one before approving example outputs.</li>"
artifact_design_items = "".join(
(
"<li>"
f"<strong>{html.escape(item.get('label', item.get('key', 'Artifact')))}</strong><br>"
f"<span>{html.escape(item.get('direction', ''))}</span>"
"</li>"
)
for item in artifact_design.get("artifact_families", [])[:3]
)
if not artifact_design_items:
artifact_design_items = "<li>No artifact design profile attached yet. Generate one before approving visual or document outputs.</li>"
design_gate_items = "".join(
f"<li>{html.escape(item)}</li>" for item in artifact_design.get("quality_gates", [])[:5]
) or "<li>No artifact design quality gates attached yet.</li>"
readiness_html = "".join(
(
"<li>"
@@ -757,6 +790,18 @@ def render_html(report: dict) -> str:
</div>
</section>
<section class="grid">
<div class="panel">
<h2>Artifact design profile</h2>
<p class="minor">Design system: {html.escape(str(artifact_design.get('design_system', 'not generated')))}</p>
<ul>{artifact_design_items}</ul>
</div>
<div class="panel">
<h2>Visual quality gates</h2>
<ul>{design_gate_items}</ul>
</div>
</section>
<section class="grid">
<div class="panel">
<h2>Reference coach</h2>
scripts/render_skill_overview.py
+34
View File
@@ -74,6 +74,10 @@ def load_reference_synthesis(skill_dir: Path) -> dict:
return load_json(skill_dir / "reports" / "reference-synthesis.json")
def load_artifact_design(skill_dir: Path) -> dict:
return load_json(skill_dir / "reports" / "artifact-design-profile.json")
def extract_title(body: str, fallback: str) -> str:
for line in body.splitlines():
if line.startswith("# "):
@@ -212,6 +216,15 @@ def synthesis_highlights(synthesis: dict) -> list[str]:
return synthesis.get("synthesis", {}).get("borrow_now", [])[:3]
def artifact_design_highlights(profile: dict) -> list[str]:
primary = profile.get("primary_artifact", {})
highlights = []
if primary.get("direction"):
highlights.append(primary["direction"])
highlights.extend(profile.get("quality_gates", [])[:3])
return highlights[:4]
def build_summary(skill_dir: Path) -> dict:
skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8")
frontmatter, body = parse_frontmatter(skill_text)
@@ -230,6 +243,7 @@ def build_summary(skill_dir: Path) -> dict:
strengths = derive_strengths(skill_dir, manifest)
benchmark = load_benchmark(skill_dir)
reference_synthesis = load_reference_synthesis(skill_dir)
artifact_design = load_artifact_design(skill_dir)
return {
"name": name,
@@ -244,6 +258,11 @@ def build_summary(skill_dir: Path) -> dict:
"introduction": introduction_lines(description),
"benchmark_highlights": benchmark_highlights(benchmark),
"synthesis_highlights": synthesis_highlights(reference_synthesis),
"artifact_design": {
"design_system": artifact_design.get("design_system", "content-led editorial"),
"primary_label": artifact_design.get("primary_artifact", {}).get("label", "General artifact"),
"highlights": artifact_design_highlights(artifact_design),
},
"metadata": {
"canonical_format": interface_data.get("compatibility", {}).get("canonical_format", "agent-skills"),
"targets": interface_data.get("compatibility", {}).get("adapter_targets", []),
@@ -288,6 +307,8 @@ def render_html(summary: dict) -> str:
for item in summary.get("benchmark_highlights", [])
)
synthesis_html = "".join(f"<li>{html.escape(item)}</li>" for item in summary.get("synthesis_highlights", []))
artifact_design = summary.get("artifact_design", {})
artifact_design_html = "".join(f"<li>{html.escape(item)}</li>" for item in artifact_design.get("highlights", []))
return f"""<!DOCTYPE html>
<html lang="en">
@@ -519,6 +540,19 @@ def render_html(summary: dict) -> str:
<ul class="strengths">{synthesis_html or "<li>No synthesis has been generated yet. Run the reference synthesis after the benchmark scan.</li>"}</ul>
</div>
</section>
<section>
<div class="section-head">
<div>
<h2>Artifact design direction</h2>
<p>Use this to understand how the generated reports, tutorials, viewers, or visual artifacts should feel before changing the layout.</p>
</div>
<div>
<p><strong>{html.escape(str(artifact_design.get("primary_label", "General artifact")))}</strong> · {html.escape(str(artifact_design.get("design_system", "content-led editorial")))}</p>
<ul class="strengths">{artifact_design_html or "<li>No artifact design profile has been generated yet.</li>"}</ul>
</div>
</div>
</section>
</div>
</body>
</html>
scripts/yao.py
+23
View File
@@ -536,6 +536,7 @@ def command_quickstart(args: argparse.Namespace) -> int:
"artifacts": {
"benchmark_scan": payload.get("artifacts", {}).get("github_benchmark_scan_md"),
"reference_synthesis": payload.get("artifacts", {}).get("reference_synthesis_md"),
"artifact_design_profile": payload.get("artifacts", {}).get("artifact_design_profile_md"),
"review_viewer": payload.get("artifacts", {}).get("review_viewer_html"),
},
},
@@ -648,6 +649,7 @@ def command_report(args: argparse.Namespace) -> int:
run_script("render_context_reports.py", []),
run_script("render_portability_report.py", []),
run_script("render_reference_synthesis.py", [str(ROOT)]),
run_script("render_artifact_design_profile.py", [str(ROOT)]),
]
)
report = {
@@ -664,6 +666,7 @@ def command_report(args: argparse.Namespace) -> int:
"context_budget": "reports/context_budget.json",
"portability_score": "reports/portability_score.json",
"reference_synthesis": "reports/reference-synthesis.json",
"artifact_design_profile": "reports/artifact-design-profile.json",
},
}
print(json.dumps(report, ensure_ascii=False, indent=2))
@@ -777,6 +780,17 @@ def command_output_risk_profile(args: argparse.Namespace) -> int:
return 0 if result["ok"] else 2
def command_artifact_design_profile(args: argparse.Namespace) -> int:
cmd = [str(Path(args.skill_dir).resolve())]
if args.output_md:
cmd.extend(["--output-md", args.output_md])
if args.output_json:
cmd.extend(["--output-json", args.output_json])
result = run_script("render_artifact_design_profile.py", cmd)
print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2))
return 0 if result["ok"] else 2
def command_iteration_directions(args: argparse.Namespace) -> int:
skill_dir = str(Path(args.skill_dir).resolve())
cmd = [skill_dir]
@@ -1145,6 +1159,15 @@ def build_parser() -> argparse.ArgumentParser:
output_risk_cmd.add_argument("--output-json")
output_risk_cmd.set_defaults(func=command_output_risk_profile)
artifact_design_cmd = subparsers.add_parser(
"artifact-design-profile",
help="Render artifact design direction and visual quality gates for a skill package.",
)
artifact_design_cmd.add_argument("skill_dir", nargs="?", default=".")
artifact_design_cmd.add_argument("--output-md")
artifact_design_cmd.add_argument("--output-json")
artifact_design_cmd.set_defaults(func=command_artifact_design_profile)
iteration_directions_cmd = subparsers.add_parser(
"iteration-directions",
help="Render the top three next iteration directions for a skill package.",
tests/verify_artifact_design_profile.py
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
def main() -> None:
tmp_root = ROOT / "tests" / "tmp_artifact_design_profile"
if tmp_root.exists():
subprocess.run(["rm", "-rf", str(tmp_root)], check=True)
tmp_root.mkdir(parents=True, exist_ok=True)
subprocess.run(
[
sys.executable,
str(ROOT / "scripts" / "init_skill.py"),
"visual-report-demo",
"--description",
"Create polished HTML reports and Markdown tutorials with screenshots, comparison tables, and reviewer notes.",
"--output-dir",
str(tmp_root),
"--intent-primary-output",
"A polished HTML report and Markdown tutorial.",
"--intent-real-input",
"screenshots",
"--intent-real-input",
"comparison data",
],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
skill_dir = tmp_root / "visual-report-demo"
proc = subprocess.run(
[sys.executable, str(ROOT / "scripts" / "render_artifact_design_profile.py"), str(skill_dir)],
cwd=ROOT,
check=True,
capture_output=True,
text=True,
)
payload = json.loads(proc.stdout)
summary = payload["summary"]
keys = {item["key"] for item in summary["artifact_families"]}
assert "report" in keys, summary
assert "tutorial" in keys, summary
assert summary["quality_gates"], summary
assert summary["layout_patterns"], summary
assert any("parchment" in item for item in summary["anti_patterns"]), summary
markdown = (skill_dir / "reports" / "artifact-design-profile.md").read_text(encoding="utf-8")
assert "Artifact Design Profile" in markdown, markdown[:500]
assert "Quality Gates" in markdown, markdown[:1200]
assert "Do not copy Kami" in markdown, markdown[:2000]
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
tests/verify_review_viewer.py
+2
View File
@@ -48,6 +48,8 @@ def main() -> None:
assert "Variant diff studio" in html_text, html_text[:900]
assert "Evidence readiness" in html_text, html_text[:1200]
assert "Output risk profile" in html_text, html_text[:1600]
assert "Artifact design profile" in html_text, html_text[:2200]
assert "Visual quality gates" in html_text, html_text[:2400]
assert "Reference coach" in html_text, html_text[:900]
assert "Reference synthesis" in html_text, html_text[:1200]
assert "Top three next moves" in html_text, html_text[:500]
tests/verify_skill_overview.py
+3
View File
@@ -52,6 +52,8 @@ def main() -> None:
assert (created / "reports" / "skill-overview.json").exists(), created
assert (created / "reports" / "reference-synthesis.md").exists(), created
assert (created / "reports" / "reference-synthesis.json").exists(), created
assert (created / "reports" / "artifact-design-profile.md").exists(), created
assert (created / "reports" / "artifact-design-profile.json").exists(), created
assert (created / "reports" / "iteration-directions.md").exists(), created
assert (created / "reports" / "iteration-directions.json").exists(), created
@@ -66,6 +68,7 @@ def main() -> None:
assert "How to introduce this skill" in report_html, report_html[:900]
assert "Patterns worth borrowing now" in report_html, report_html[:1200]
assert "Reference synthesis" in report_html, report_html[:1500]
assert "Artifact design direction" in report_html, report_html[:2000]
intent_text = (created / "reports" / "intent-dialogue.md").read_text(encoding="utf-8")
assert "Questions To Ask" in intent_text, intent_text[:400]
tests/verify_yao_cli.py
+8
View File
@@ -53,6 +53,7 @@ def main() -> None:
assert (created / "reports" / "reference-scan.md").exists(), created
assert (created / "reports" / "reference-synthesis.md").exists(), created
assert (created / "reports" / "output-risk-profile.md").exists(), created
assert (created / "reports" / "artifact-design-profile.md").exists(), created
assert (created / "reports" / "iteration-directions.md").exists(), created
assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created
@@ -82,6 +83,7 @@ def main() -> None:
assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root
assert (quickstart_root / "reports" / "intent-confidence.md").exists(), quickstart_root
assert (quickstart_root / "reports" / "reference-synthesis.md").exists(), quickstart_root
assert (quickstart_root / "reports" / "artifact-design-profile.md").exists(), quickstart_root
assert quickstart_result["payload"]["archetype"] == "production", quickstart_result
assert quickstart_result["payload"]["guidance"]["experience_note"], quickstart_result
assert quickstart_result["payload"]["guidance"]["problem_diagnosis"]["candidates"], quickstart_result
@@ -172,6 +174,11 @@ def main() -> None:
assert output_risk_result["payload"]["artifacts"]["markdown"].endswith("reports/output-risk-profile.md"), output_risk_result
assert output_risk_result["payload"]["summary"]["risk_families"], output_risk_result
artifact_design_result = run("artifact-design-profile", str(created))
assert artifact_design_result["ok"], artifact_design_result
assert artifact_design_result["payload"]["artifacts"]["markdown"].endswith("reports/artifact-design-profile.md"), artifact_design_result
assert artifact_design_result["payload"]["summary"]["quality_gates"], artifact_design_result
directions_result = run("iteration-directions", str(created))
assert directions_result["ok"], directions_result
assert directions_result["payload"]["artifacts"]["markdown"].endswith("reports/iteration-directions.md"), directions_result
@@ -219,6 +226,7 @@ def main() -> None:
assert report_result["ok"], report_result
assert "iteration_ledger" in report_result["payload"]["artifacts"], report_result
assert "portability_score" in report_result["payload"]["artifacts"], report_result
assert "artifact_design_profile" in report_result["payload"]["artifacts"], report_result
package_dir = tmp_root / "dist"
package_result = run("package", ".", "--platform", "generic", "--output-dir", str(package_dir))