From e39be9d5d40513f362d657b81846b48ce140673b Mon Sep 17 00:00:00 2001 From: yaojingang Date: Wed, 17 Jun 2026 15:53:58 +0800 Subject: [PATCH] Refresh clean release evidence reports --- reports/adaptation_proposals.json | 2 +- reports/adaptation_proposals.md | 2 +- reports/adoption_drift_report.json | 28 +- reports/adoption_drift_report.md | 10 +- reports/architecture_maintainability.json | 2 +- reports/architecture_maintainability.md | 2 +- reports/benchmark_reproducibility.json | 157 +- reports/benchmark_reproducibility.md | 66 +- reports/compiled_targets.json | 2 +- reports/context_budget.json | 20 +- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 12 +- reports/evidence_consistency.json | 310 +-- reports/python_compatibility.json | 2 +- reports/python_compatibility.md | 2 +- reports/review-studio.html | 38 +- reports/review-studio.json | 2319 +++++++++---------- reports/review-viewer.html | 18 +- reports/review-viewer.json | 1566 ++++++------- reports/review_annotations.json | 2 +- reports/review_waivers.json | 4 +- reports/review_waivers.md | 2 +- reports/skill-interpretation.html | 22 +- reports/skill-interpretation.json | 230 +- reports/skill-overview.html | 22 +- reports/skill-overview.json | 230 +- reports/skill_os2_audit.json | 6 +- reports/skill_os2_audit.md | 4 +- reports/skillops/daily/2026-06-17.json | 8 +- reports/skillops/daily/2026-06-17.md | 4 +- reports/skillops/weekly/2026-W25.json | 8 +- reports/skillops/weekly/2026-W25.md | 4 +- reports/world_class_evidence_intake.json | 2 +- reports/world_class_evidence_ledger.json | 20 +- reports/world_class_evidence_ledger.md | 12 +- reports/world_class_evidence_plan.json | 4 +- reports/world_class_evidence_plan.md | 2 +- reports/world_class_evidence_preflight.html | 97 +- reports/world_class_evidence_preflight.json | 350 ++- reports/world_class_evidence_preflight.md | 25 +- reports/world_class_operator_runbook.html | 12 +- reports/world_class_operator_runbook.json | 97 +- reports/world_class_operator_runbook.md | 15 +- reports/world_class_submission_review.json | 18 +- reports/world_class_submission_review.md | 2 +- 45 files changed, 2695 insertions(+), 3067 deletions(-) diff --git a/reports/adaptation_proposals.json b/reports/adaptation_proposals.json index 114b1545..deae6ac2 100644 --- a/reports/adaptation_proposals.json +++ b/reports/adaptation_proposals.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T07:47:34Z", + "generated_at": "2026-06-17T07:53:10Z", "skill_dir": ".", "source_patterns": "reports/user_patterns.json", "pattern_count": 5, diff --git a/reports/adaptation_proposals.md b/reports/adaptation_proposals.md index 1ca31c13..23e71e3a 100644 --- a/reports/adaptation_proposals.md +++ b/reports/adaptation_proposals.md @@ -1,6 +1,6 @@ # Adaptation Proposals -- Generated at: `2026-06-17T07:47:34Z` +- Generated at: `2026-06-17T07:53:10Z` - Pattern report: `reports/user_patterns.json` - Proposal only: `true` - Writes repository files: `false` diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index a6291f56..f4158b0a 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-17T07:48:13Z", + "generated_at": "2026-06-17T07:53:09Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -25,14 +25,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -41,7 +41,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -53,31 +53,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "skill_activation", + "event": "review_event", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } diff --git a/reports/adoption_drift_report.md b/reports/adoption_drift_report.md index 206372c9..91b31983 100644 --- a/reports/adoption_drift_report.md +++ b/reports/adoption_drift_report.md @@ -5,9 +5,9 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Summary - Events: `1` -- Adoption samples: `1` -- Activation events: `1` -- Adoption rate: `100.0` +- Adoption samples: `0` +- Activation events: `0` +- Adoption rate: `0` - Missed trigger signals: `0` - Bad output signals: `0` - Script error signals: `0` @@ -25,7 +25,7 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, | Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| `yao-meta-skill` | 1 | 1 | 1 | 0 | 0 | 0 | 100.0 | +| `yao-meta-skill` | 1 | 0 | 0 | 0 | 0 | 0 | 0 | ## Next Iteration Candidates @@ -33,4 +33,4 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Recent Metadata Events -- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` source=`manual` command=`unknown` activation=`explicit` outcome=`accepted` failure=`none` +- `2026-06-13T12:00:00Z` `yao-meta-skill` event=`review_event` source=`manual` command=`unknown` activation=`manual` outcome=`reviewed` failure=`none` diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index dd5b97b1..07c883ca 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "python_file_count": 229, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 3f7d9dca..f254d8ff 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -1,6 +1,6 @@ # Architecture Maintainability -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 0c8dbec5..77c8b6ec 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,50 +3,43 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "ff0473ac02f14a3a431cb01f157453335eb14901", + "commit": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "git_status": { "available": true, "dirty": true, - "changed_file_count": 64, + "changed_file_count": 45, "generated_dirty": true, - "generated_changed_file_count": 58, - "source_dirty": true, - "source_changed_file_count": 6, + "generated_changed_file_count": 45, + "source_dirty": false, + "source_changed_file_count": 0, "sample": [ - " M evidence/world_class/README.md", - " M registry/index.json", - " M registry/packages/yao-meta-skill.json", - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/adoption_drift_report.json", - " M reports/architecture_maintainability.json", - " M reports/architecture_maintainability.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", - " M reports/compiled_targets.json", - " M reports/context_budget.json" - ], - "source_sample": [ - " M evidence/world_class/README.md", - " M scripts/evidence_consistency_world_class.py", - " M scripts/render_world_class_operator_runbook.py", - " M tests/verify_evidence_consistency.py", - " M tests/verify_world_class_operator_runbook.py", - "?? scripts/world_class_operator_runbook_coordination.py" - ], - "generated_sample": [ - " M registry/index.json", - " M registry/packages/yao-meta-skill.json", " M reports/adaptation_proposals.json", " M reports/adaptation_proposals.md", " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", " M reports/architecture_maintainability.json", " M reports/architecture_maintainability.md", " M reports/benchmark_reproducibility.json", " M reports/benchmark_reproducibility.md", " M reports/compiled_targets.json", " M reports/context_budget.json", - " M reports/context_budget.md" + " M reports/context_budget.md", + " M reports/context_budget_summary.json" + ], + "source_sample": [], + "generated_sample": [ + " M reports/adaptation_proposals.json", + " M reports/adaptation_proposals.md", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json" ], "generated_dirty_prefixes": [ "dist/", @@ -60,57 +53,55 @@ }, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d", - "source_contract_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, "command_executed_count": 10, "timing_observed_count": 10, - "model_executed_count": 0, - "token_observed_count": 0, + "model_executed_count": 10, + "token_observed_count": 10, "human_review_complete": false, - "provider_evidence_complete": false, + "provider_evidence_complete": true, "world_class_ready": false, - "world_class_open_gap_count": 4, + "world_class_open_gap_count": 3, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5, + "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 64, - "source_tree_dirty": true, - "source_changed_file_count": 6, + "changed_file_count": 45, + "source_tree_dirty": false, + "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 58 + "generated_changed_file_count": 45 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ - "release lock is not clean or commit is unavailable", - "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", - "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (10/19 pass, 9 blocked)" + "world-class evidence is not accepted yet (3 open gaps, 4 ledger pending)", + "world-class source checks are not all accepted (11/19 pass, 8 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": false, - "commit": "ff0473ac02f14a3a431cb01f157453335eb14901", + "ready": true, + "commit": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "status_scope": "generation-time status before this report is written", - "source_changed_file_count": 6, - "generated_changed_file_count": 58, - "reason": "source files were dirty at generation time" + "source_changed_file_count": 0, + "generated_changed_file_count": 45, + "reason": "only generated evidence artifacts were dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -118,7 +109,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d" + "sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -191,8 +182,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7967, - "sha256": "f02f5250076a368521a463325fe1510f2d4334d492464a5270c9f94fab3b9662" + "bytes": 8449, + "sha256": "4df66b63d2e717a559a6881b966fb1b92def7329cc89aa48ee0d8596f87402e5" }, { "label": "blind_review", @@ -212,8 +203,8 @@ "label": "trigger_scorecard", "path": "reports/route_scorecard.json", "exists": true, - "bytes": 16961, - "sha256": "c164e83e36d0af276b6af2de2a5e026d7f0711b83eef2b5dcd0e760bd8bb28fc" + "bytes": 16950, + "sha256": "a649547a7d3d5fe02262ca1b188ed4022d8f557770be425f7b7cf762aa76533a" }, { "label": "runtime_conformance", @@ -226,22 +217,22 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 137649, - "sha256": "0873eba396f79089e01d15d6e61046e4749a8876bc13c2c604105ea282cbc1e8" + "bytes": 137787, + "sha256": "76e0d31fe1e0e0c6f96eaa2321733ca456d8c59d3eb9007a941bb3d8afdbc02e" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, "bytes": 30127, - "sha256": "2bb4f894fdd29bd3b7714d524e9e9bcee6a4c40d948a34d89a15829acf2be9f7" + "sha256": "d31dc03c74f17242f4163b8cb3f23120fe3d0712edd08fa183cb94f8120d491c" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, - "bytes": 3183, - "sha256": "5132d2d7420f4b0fe09f09fc75756865695a1249564c8ab0356deef8fe7c55ab" + "bytes": 3137, + "sha256": "49e2aee07e38a83608504a3a4f79dc07889e4ba899c4b96859721a5a7cb0a856" }, { "label": "package_verification", @@ -255,70 +246,70 @@ "path": "reports/install_simulation.json", "exists": true, "bytes": 8758, - "sha256": "91c2bd418918f2877596e75b986c4ee6c55a50bf00063577be94ae65fc8399f5" + "sha256": "c1f1122264b868c4998482c580ef94333575c6ff7c5c9c297188b225f69a131d" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, - "bytes": 14466, - "sha256": "f68822068601b254ebdf8966c74386ca15845eabcb93b3e82355931ab4651670" + "bytes": 13824, + "sha256": "6362e79a8cfd3ff54b1a7ca32f37de8ed1cda500b953fdf1074928c6d2d62379" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, - "bytes": 22784, - "sha256": "d0ab417742950ff5a5f3407f068829217da5c997a6a7c5a5ee59441cb4abfb7f" + "bytes": 23314, + "sha256": "8eb5e1fbaa7580f107fd9920d13a13b239edb72d50b6959fc8f6dd43150f07d7" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 26016, - "sha256": "fc4c3e398ad0de110ae5f37e461d03712716552ce85b44e41e09f4d343a17f0b" + "bytes": 26940, + "sha256": "a345880034709fcad8741cf20114dba739e2652a76e65d58265f8a87d10c8698" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 20646, - "sha256": "1a939bdd1f4a3ee30e05f0b286145f0f3320ba9f1935749806a581737155774a" + "bytes": 21720, + "sha256": "ebbfa25a5211125583f333fdd41a3dea936a54350cc6361dee751fa1da46881b" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 97462, - "sha256": "d6c13c020987c1b395d8e78cc409fec0784f6746ff77f0ef148abf105b9d6045" + "bytes": 94331, + "sha256": "eecbe8b0eed0581af20140cfdccbf4346b9b0ca8022f4a033940a166c2415195" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 17299, - "sha256": "b256c40e75283556fc2ab1a0ff3a19834e82d54d4c7a3e7464806c01aab69db7" + "bytes": 17725, + "sha256": "56f927077bbdb0f601dc56f0d8645432db8e9a8ad78e323828a91dd2f52a1f1b" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 90823, - "sha256": "f9aa6f5c13af9ee6f5dcb11dafe66b730e5ad41445acf826712b2abef6703a0d" + "bytes": 87920, + "sha256": "94cce92c415b0348348c9b7dbe9eaf233a1b64ff37b5b98e128e11f2f40412ea" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 25405, - "sha256": "f77de0ede45edb4793e4648cdf164bd9caf09e406fd42764f4221a623df175c3" + "bytes": 25549, + "sha256": "94a63a39d2d0989db7a0c6d15f49a5027148654ab30fd197205e7b78173be262" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 34524, - "sha256": "6e07cc2c00a37ae522b7fa1afa09bcb9b6eddf926979582cd2d24fbc9476631f" + "bytes": 34702, + "sha256": "4dc91fdfc2e54a83aedf5ea99e80481949326d74c6933abd9d85c5f4a8f10c5b" }, { "label": "world_class_claim_guard", @@ -453,7 +444,7 @@ }, "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", - "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", "Pending blind-review decisions are visible but do not count as human adjudication.", "World-class readiness remains false until external and human evidence gaps close." ], diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index cab6b0b0..29d45da7 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,33 +1,33 @@ # Benchmark Reproducibility Generated at: `2026-06-17` -Commit: `ff0473ac02f14a3a431cb01f157453335eb14901` +Commit: `cece6f9b04869e46db9aed1bf0206ccb8fb887e4` Working tree dirty at generation: `true` -Source tree dirty at generation: `true` +Source tree dirty at generation: `false` Generated evidence dirty at generation: `true` -Evidence bundle SHA256: `9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d` +Evidence bundle SHA256: `53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350` ## Summary - reproducibility ready: `true` -- release lock ready: `false` +- release lock ready: `true` - methodology complete: `true` - required artifacts: `25` - missing artifacts: `0` -- source contract sha256: `795846775173` +- source contract sha256: `7fc324e26d61` - archive sha256: `7db44f059844` - output cases: `5` - disclosed failure cases: `3` - reproduction commands: `23` -- provider evidence complete: `false` +- provider evidence complete: `true` - human review complete: `false` - world-class ready: `false` -- world-class source checks: `10` pass / `19` total; `9` blocked +- world-class source checks: `11` pass / `19` total; `8` blocked - public claim ready: `false` -- public claim blockers: `5` -- changed files at generation: `64` -- source changed files at generation: `6` -- generated changed files at generation: `58` +- public claim blockers: `3` +- changed files at generation: `45` +- source changed files at generation: `0` +- generated changed files at generation: `45` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -39,23 +39,21 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | -| release lock is not clean or commit is unavailable | -| provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | -| world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | -| world-class source checks are not all accepted (10/19 pass, 9 blocked) | +| world-class evidence is not accepted yet (3 open gaps, 4 ledger pending) | +| world-class source checks are not all accepted (11/19 pass, 8 blocked) | ## Release Lock -- ready: `false` -- reason: source files were dirty at generation time +- ready: `true` +- reason: only generated evidence artifacts were dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d` +- sha256: `53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350` ## Methodology Sections @@ -77,25 +75,25 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `f02f5250076a` | +| output_execution | `reports/output_execution_runs.json` | present | `4df66b63d2e7` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `91fd88dd9b0f` | -| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | +| trigger_scorecard | `reports/route_scorecard.json` | present | `a649547a7d3d` | | runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | -| trust_report | `reports/security_trust_report.json` | present | `0873eba396f7` | -| python_compatibility | `reports/python_compatibility.json` | present | `2bb4f894fdd2` | -| registry_audit | `reports/registry_audit.json` | present | `5132d2d7420f` | +| trust_report | `reports/security_trust_report.json` | present | `76e0d31fe1e0` | +| python_compatibility | `reports/python_compatibility.json` | present | `d31dc03c74f1` | +| registry_audit | `reports/registry_audit.json` | present | `49e2aee07e38` | | package_verification | `reports/package_verification.json` | present | `7aa6c25e8154` | -| install_simulation | `reports/install_simulation.json` | present | `91c2bd418918` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `f68822068601` | -| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `d0ab41774295` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `fc4c3e398ad0` | -| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `1a939bdd1f4a` | -| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `d6c13c020987` | -| world_class_submission_review | `reports/world_class_submission_review.json` | present | `b256c40e7528` | -| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `f9aa6f5c13af` | -| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `f77de0ede45e` | -| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `6e07cc2c00a3` | +| install_simulation | `reports/install_simulation.json` | present | `c1f1122264b8` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `6362e79a8cfd` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `8eb5e1fbaa75` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `a34588003470` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `ebbfa25a5211` | +| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `eecbe8b0eed0` | +| world_class_submission_review | `reports/world_class_submission_review.json` | present | `56f927077bbd` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `94cce92c415b` | +| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `94a63a39d2d0` | +| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `4dc91fdfc2e5` | | world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `abe7f7d60c00` | ## Reproduction Commands @@ -156,6 +154,6 @@ This report proves local benchmark reproducibility only. It keeps external provi ## Limits - The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately. -- Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence. +- Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet. - Pending blind-review decisions are visible but do not count as human adjudication. - World-class readiness remains false until external and human evidence gaps close. diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index a326ac10..f9f14ed0 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "target_count": 5, diff --git a/reports/context_budget.json b/reports/context_budget.json index 15aa489e..d7b036c3 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -5,16 +5,16 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 797, - "other_text_tokens": 1151006, - "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1151803, - "deferred_resource_tokens": 521572, + "skill_body_tokens": 805, + "other_text_tokens": 1152733, + "estimated_initial_load_tokens": 998, + "estimated_total_text_tokens": 1153538, + "deferred_resource_tokens": 523653, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 460269, + "estimated_tokens": 462350, "file_count": 153 }, { @@ -36,7 +36,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 460269, + "estimated_tokens": 462350, "file_count": 153 } ], @@ -59,7 +59,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 460269, + "estimated_tokens": 462350, "file_count": 153, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } @@ -68,7 +68,7 @@ }, "relevant_file_count": 712, "unused_resource_dirs": [], - "quality_signal_points": 130, - "quality_density": 131.3 + "quality_signal_points": 140, + "quality_density": 140.3 } } diff --git a/reports/context_budget.md b/reports/context_budget.md index 2b015908..a6e4d386 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 521572 | `governed` | scripts:460269 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 998 | 805 | 523653 | `governed` | scripts:462350 | 140.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 5fdb5c90..c3dd2f93 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -6,13 +6,13 @@ "path": ".", "budget_tier": "production", "budget_limit": 1000, - "initial_tokens": 990, - "skill_body_tokens": 797, - "deferred_resource_tokens": 521572, + "initial_tokens": 998, + "skill_body_tokens": 805, + "deferred_resource_tokens": 523653, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 460269, + "estimated_tokens": 462350, "file_count": 153 } ], @@ -35,14 +35,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 460269, + "estimated_tokens": 462350, "file_count": 153, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "quality_density": 131.3, + "quality_density": 140.3, "unused_resource_dirs": [], "ok": true, "warnings": [] diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 1c5aeffd..3494d9ec 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -324,12 +324,12 @@ "status": "pass", "expected": { "status": "pass", - "detail": "initial load 990/1000; deferred 521572/120000; top deferred scripts 460269; resource governance governed; quality density 131.3", + "detail": "initial load 998/1000; deferred 523653/120000; top deferred scripts 462350; resource governance governed; quality density 140.3", "evidence": "reports/context_budget.json" }, "actual": { "status": "pass", - "detail": "initial load 990/1000; deferred 521572/120000; top deferred scripts 460269; resource governance governed; quality density 131.3", + "detail": "initial load 998/1000; deferred 523653/120000; top deferred scripts 462350; resource governance governed; quality density 140.3", "evidence": "reports/context_budget.json" }, "paths": [ @@ -342,8 +342,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches source dirty state", "status": "pass", - "expected": false, - "actual": false, + "expected": true, + "actual": true, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -357,7 +357,7 @@ "actual": { "available": true, "clean": false, - "changed_file_count": 60 + "changed_file_count": 45 }, "paths": [ "reports/benchmark_reproducibility.json" @@ -398,8 +398,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "ff0473ac02f14a3a431cb01f157453335eb14901", - "actual": "ff0473ac02f14a3a431cb01f157453335eb14901", + "expected": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", + "actual": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -411,30 +411,30 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": false, + "release_lock_ready": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "ae477eea3fa4a0fc5720e982b5382dcb245af7d36cef514563bf0e2776811c8f", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 9, - "world_class_source_blocked_count": 10, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5 + "public_claim_blocker_count": 3 }, "actual": { - "release_lock_ready": false, + "release_lock_ready": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "ae477eea3fa4a0fc5720e982b5382dcb245af7d36cef514563bf0e2776811c8f", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 9, - "world_class_source_blocked_count": 10, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5 + "public_claim_blocker_count": 3 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -491,8 +491,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 9, - "source_blocked_count": 10, + "source_pass_count": 11, + "source_blocked_count": 8, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -503,8 +503,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 9, - "source_blocked_count": 10, + "source_pass_count": 11, + "source_blocked_count": 8, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -524,7 +524,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 19, - "source_pass_count": 9 + "source_pass_count": 11 }, "actual": { "ready": false, @@ -532,7 +532,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 19, - "source_pass_count": 9 + "source_pass_count": 11 }, "paths": [ "reports/world_class_evidence_ledger.json", @@ -544,8 +544,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "ff0473ac02f14a3a431cb01f157453335eb14901", - "actual": "ff0473ac02f14a3a431cb01f157453335eb14901", + "expected": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", + "actual": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -557,30 +557,30 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": false, + "release_lock_ready": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "ae477eea3fa4a0fc5720e982b5382dcb245af7d36cef514563bf0e2776811c8f", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 9, - "world_class_source_blocked_count": 10, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5 + "public_claim_blocker_count": 3 }, "actual": { - "release_lock_ready": false, + "release_lock_ready": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "ae477eea3fa4a0fc5720e982b5382dcb245af7d36cef514563bf0e2776811c8f", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 9, - "world_class_source_blocked_count": 10, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5 + "public_claim_blocker_count": 3 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -637,8 +637,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 9, - "source_blocked_count": 10, + "source_pass_count": 11, + "source_blocked_count": 8, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -649,8 +649,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 9, - "source_blocked_count": 10, + "source_pass_count": 11, + "source_blocked_count": 8, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -670,7 +670,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 19, - "source_pass_count": 9 + "source_pass_count": 11 }, "actual": { "ready": false, @@ -678,7 +678,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 19, - "source_pass_count": 9 + "source_pass_count": 11 }, "paths": [ "reports/world_class_evidence_ledger.json", @@ -727,7 +727,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 360 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -748,7 +748,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 360 个词/字，references 约 16934 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -791,7 +791,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 360 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -812,7 +812,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 360 个词/字，references 约 16934 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -942,7 +942,7 @@ "status": "pass", "expected": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -970,7 +970,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -984,7 +984,7 @@ }, "actual": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -1012,7 +1012,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -1044,7 +1044,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -1054,10 +1054,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -1092,7 +1092,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -1102,10 +1102,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -1167,7 +1167,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 360 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -1326,7 +1326,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 360 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -1477,7 +1477,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 9, + "source_pass_count": 11, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1491,10 +1491,7 @@ "status": "pending", "summary_zh": "缺少真实 provider 模型运行和 token metadata。", "summary_en": "Missing a real provider model run and token metadata.", - "blocked_checks": [ - "Provider model run", - "Token usage observed" - ] + "blocked_checks": [] }, { "key": "human-adjudication", @@ -1552,7 +1549,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 9, + "source_pass_count": 11, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1566,10 +1563,7 @@ "status": "pending", "summary_zh": "缺少真实 provider 模型运行和 token metadata。", "summary_en": "Missing a real provider model run and token metadata.", - "blocked_checks": [ - "Provider model run", - "Token usage observed" - ] + "blocked_checks": [] }, { "key": "human-adjudication", @@ -1963,15 +1957,15 @@ "expected": { "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 9, - "world_class_source_blocked_count": 10, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false }, "actual": { "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 9, - "world_class_source_blocked_count": 10, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false }, "paths": [ @@ -1987,8 +1981,8 @@ "expected": { "pending_count": 4, "source_check_count": 19, - "source_pass_count": 9, - "source_blocked_count": 10, + "source_pass_count": 11, + "source_blocked_count": 8, "ready_to_claim_world_class": false, "preflight_counts_as_evidence": false, "credential_value_exposed": false @@ -1996,8 +1990,8 @@ "actual": { "pending_count": 4, "source_check_count": 19, - "source_pass_count": 9, - "source_blocked_count": 10, + "source_pass_count": 11, + "source_blocked_count": 8, "ready_to_claim_world_class": false, "preflight_counts_as_evidence": false, "credential_value_exposed": false @@ -2174,7 +2168,7 @@ "summary": { "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 14, + "phase_queue_row_count": 12, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2207,19 +2201,17 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 10, - "row_count": 10, + "blocked_count": 8, + "row_count": 8, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", - "operator with provider credentials", "target client or installer integrator" ], "evidence_keys": [ "human-adjudication", "native-client-telemetry", - "native-permission-enforcement", - "provider-holdout" + "native-permission-enforcement" ], "next_action_id": "human-adjudication-source-check-blind_review_attested", "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2240,25 +2232,9 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", + "next_action_id": "provider-holdout-precheck-provider-api-key", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false - }, - { - "phase": "collect-source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false } ], "human-adjudication": [ @@ -2368,7 +2344,7 @@ "operator_runbook_summary": { "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 14, + "phase_queue_row_count": 12, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2401,19 +2377,17 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 10, - "row_count": 10, + "blocked_count": 8, + "row_count": 8, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", - "operator with provider credentials", "target client or installer integrator" ], "evidence_keys": [ "human-adjudication", "native-client-telemetry", - "native-permission-enforcement", - "provider-holdout" + "native-permission-enforcement" ], "next_action_id": "human-adjudication-source-check-blind_review_attested", "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2434,25 +2408,9 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", + "next_action_id": "provider-holdout-precheck-provider-api-key", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false - }, - { - "phase": "collect-source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false } ], "human-adjudication": [ @@ -2572,25 +2530,9 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", + "next_action_id": "provider-holdout-precheck-provider-api-key", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false - }, - { - "phase": "collect-source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false } ], "human-adjudication": [ @@ -2701,7 +2643,7 @@ "summary": { "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 14, + "phase_queue_row_count": 12, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2734,19 +2676,17 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 10, - "row_count": 10, + "blocked_count": 8, + "row_count": 8, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", - "operator with provider credentials", "target client or installer integrator" ], "evidence_keys": [ "human-adjudication", "native-client-telemetry", - "native-permission-enforcement", - "provider-holdout" + "native-permission-enforcement" ], "next_action_id": "human-adjudication-source-check-blind_review_attested", "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2756,7 +2696,7 @@ "operator_runbook_summary": { "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 14, + "phase_queue_row_count": 12, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2789,19 +2729,17 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 10, - "row_count": 10, + "blocked_count": 8, + "row_count": 8, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", - "operator with provider credentials", "target client or installer integrator" ], "evidence_keys": [ "human-adjudication", "native-client-telemetry", - "native-permission-enforcement", - "provider-holdout" + "native-permission-enforcement" ], "next_action_id": "human-adjudication-source-check-blind_review_attested", "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2822,25 +2760,9 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", + "next_action_id": "provider-holdout-precheck-provider-api-key", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false - }, - { - "phase": "collect-source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false } ], "human-adjudication": [ @@ -2960,25 +2882,9 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", + "next_action_id": "provider-holdout-precheck-provider-api-key", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false - }, - { - "phase": "collect-source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false } ], "human-adjudication": [ @@ -3098,25 +3004,9 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", + "next_action_id": "provider-holdout-precheck-provider-api-key", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false - }, - { - "phase": "collect-source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false } ], "human-adjudication": [ @@ -3316,8 +3206,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 9, - "source_blocked_count": 10, + "source_pass_count": 11, + "source_blocked_count": 8, "plan_keys": [ "human-adjudication", "native-client-telemetry", @@ -3474,8 +3364,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 9, - "source_blocked_count": 10, + "source_pass_count": 11, + "source_blocked_count": 8, "plan_keys": [ "human-adjudication", "native-client-telemetry", @@ -3647,7 +3537,7 @@ "`0` permission failures", "`25` required artifacts", "`23` reproduction commands", - "initial load `990/1000`", + "initial load `998/1000`", "target count is `82`" ], "actual": "all present", diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index d8fbdc63..5c81f5f1 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "root": ".", "summary": { "target_python": "3.11", diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 68ee1f0a..15ecf61f 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -1,6 +1,6 @@ # Python Compatibility -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/review-studio.html b/reports/review-studio.html index ef13ae7f..4baadfc5 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -762,7 +762,7 @@
Review Studio 2.0

Yao Meta Skill

-

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

审查结论 review @@ -772,12 +772,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

153 scripts scanned; secrets found

Py Compat0

232 files scanned for Python 3.11

Arch Debt0

706 largest lines; 0 watchlist; 5 early; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock false

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

182 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

689 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 10; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

3 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

153 scripts scanned; secrets found

Py Compat0

232 files scanned for Python 3.11

Arch Debt0

711 largest lines; 0 watchlist; 5 early; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 1 invalid

Claim Guard0

182 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

689 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 521572/120000; top deferred scripts 460269; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 153 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 232 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

229 Python files; 0 hotspots; 0 watchlist files; 5 early watch files; 0 blockers; largest 706 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 998/1000; deferred 523653/120000; top deferred scripts 462350; resource governance governed; quality density 140.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 153 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 232 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

229 Python files; 0 hotspots; 0 watchlist files; 5 early watch files; 0 blockers; largest 711 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
@@ -787,12 +787,12 @@

阻断事项

无。

-

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true
+

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md + reports/output_review_decisions.json
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
修复
3 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: operator with provider credentialsevidence: provider-holdoutSet OPENAI_API_KEY in the operator shell; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source2 / 2 blockedowners: operator with provider credentialsevidence: provider-holdoutRun provider-backed output-exec with real credentials.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckopenai-api-keyoperator with provider credentialsRequired env precheck is missing.Set OPENAI_API_KEY in the operator shell; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkmodel_executed_countoperator with provider credentialsCurrent value 0 does not satisfy '>0'.Run provider-backed output-exec with real credentials.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checktoken_observed_countoperator with provider credentialsCurrent value 0 does not satisfy '>0'.Provider execution should return non-estimated token usage.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  2. export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  3. python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
5 blocked / 5 pass
修复
6 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: human reviewerevidence: human-adjudicationAssign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source5 / 5 blockedowners: human reviewerevidence: human-adjudicationSet reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckhuman-reviewerhuman reviewerRequired human precheck is human-required.Assign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkblind_review_attestedhuman reviewerCurrent value False does not satisfy 'true'.Set reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkjudgment_counthuman reviewerCurrent value 0 does not satisfy '==pair_count'.Every pair needs one valid human judgment.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkpending_counthuman reviewerCurrent value 5 does not satisfy '==0'.Record a reviewer choice and reason for every pair.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkready_for_human_evidencehuman reviewerCurrent value False does not satisfy 'true'.Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkreviewer_metadata_presenthuman reviewerCurrent value False does not satisfy 'true'.Record reviewer and reviewed_at before adjudication can count.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
修复
2 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementAttach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementCollect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · prechecknative-guardtarget client or installer integratorRequired external precheck is external-required.Attach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checknative_enforcement_counttarget client or installer integratorCurrent value 0 does not satisfy '>0'.Collect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 1

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
1 blocked / 2 pass
修复
2 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryInstall a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source1 / 1 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryImport at least one metadata-only event from a real client.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckexternal-clientBrowser/Chrome/IDE/provider client integratorRequired external precheck is external-required.Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkexternal_source_eventsBrowser/Chrome/IDE/provider client integratorCurrent value 0 does not satisfy '>0'.Import at least one metadata-only event from a real client.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md + reports/output_review_decisions.json
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 10; token-observed 10

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
0 blocked / 3 pass
修复
1 repair rows; counts as completion: false
阶段
1 blocked phases; counts as completion: false
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查

暂无阻断检查。

阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: operator with provider credentialsevidence: provider-holdoutSet one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckprovider-api-keyoperator with provider credentialsRequired env_any precheck is missing.Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  2. For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  3. For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
5 blocked / 5 pass
修复
6 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: human reviewerevidence: human-adjudicationAssign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source5 / 5 blockedowners: human reviewerevidence: human-adjudicationSet reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckhuman-reviewerhuman reviewerRequired human precheck is human-required.Assign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkblind_review_attestedhuman reviewerCurrent value False does not satisfy 'true'.Set reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkjudgment_counthuman reviewerCurrent value 0 does not satisfy '==pair_count'.Every pair needs one valid human judgment.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkpending_counthuman reviewerCurrent value 5 does not satisfy '==0'.Record a reviewer choice and reason for every pair.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkready_for_human_evidencehuman reviewerCurrent value False does not satisfy 'true'.Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkreviewer_metadata_presenthuman reviewerCurrent value False does not satisfy 'true'.Record reviewer and reviewed_at before adjudication can count.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
修复
2 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementAttach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementCollect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · prechecknative-guardtarget client or installer integratorRequired external precheck is external-required.Attach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checknative_enforcement_counttarget client or installer integratorCurrent value 0 does not satisfy '>0'.Collect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
修复
3 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryInstall a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source2 / 2 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryTelemetry must include adoption outcome evidence.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckexternal-clientBrowser/Chrome/IDE/provider client integratorRequired external precheck is external-required.Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkadoption_sample_countBrowser/Chrome/IDE/provider client integratorCurrent value 0 does not satisfy '>0'.Telemetry must include adoption outcome evidence.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkexternal_source_eventsBrowser/Chrome/IDE/provider client integratorCurrent value 0 does not satisfy '>0'.Import at least one metadata-only event from a real client.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -819,7 +819,7 @@

输出实验

案例数
5
With Skill
100
Baseline
0
增益
100
Gate
是
-

执行证据

运行数
10
命令执行
10
模型执行
0
记录样本
0
计时样本
10
估算 Token
10
+

执行证据

运行数
10
命令执行
10
模型执行
10
记录样本
0
计时样本
10
估算 Token
0
@@ -844,12 +844,12 @@
-

上下文

initial load 990/1000; deferred 521572/120000; top deferred scripts 460269; resource governance governed; quality density 131.3

+

上下文

initial load 998/1000; deferred 523653/120000; top deferred scripts 462350; resource governance governed; quality density 140.3

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
153
网络脚本
3
Help 失败
0
包体哈希
7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291
+

信任报告

Secret
0
脚本数
153
网络脚本
3
Help 失败
0
包体哈希
7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -874,14 +874,14 @@
-

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

-

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

+

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
-

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
否
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
-

周度队列

决策
curator-review
Week Id
2026-W25
日报数
2
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
否
Writes Source Files
否
Auto Patch Enabled
否
+

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
是
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
+

周度队列

决策
curator-review
Week Id
2026-W25
日报数
2
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
是
Writes Source Files
否
Auto Patch Enabled
否
@@ -893,19 +893,19 @@

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0

批准候选

-
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 0; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
无
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .
+
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 10; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
无
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .

世界证据

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

-
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; reviewer_metadata_present: False; reason_required: True; blind_review_attested: False; raw_content_excluded_attested: True; reviewer_reason_required_attested: True; ready_for_human_evidence: False; raw_content_allowed: False; raw_content_path_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Reason requiredreason_required: True / trueKeep reason mandatory for every imported or direct reviewer decision.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Raw content attestedraw_content_excluded_attested: True / trueAttest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
  • Raw content blockedraw_content_allowed: False / falseAdjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 1
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 1; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 1 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 10; token-observed 10
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 10; timing_observed_count: 10; token_observed_count: 10; accepted: True
提交态
status: invalid-contract; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: True; privacy_contract_satisfied: True

执行步骤

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 10 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 10 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; reviewer_metadata_present: False; reason_required: True; blind_review_attested: False; raw_content_excluded_attested: True; reviewer_reason_required_attested: True; ready_for_human_evidence: False; raw_content_allowed: False; raw_content_path_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Reason requiredreason_required: True / trueKeep reason mandatory for every imported or direct reviewer decision.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Raw content attestedraw_content_excluded_attested: True / trueAttest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
  • Raw content blockedraw_content_allowed: False / falseAdjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.

提交清单

每张卡片给出模板、提交文件、准备命令、校验命令、收集要求、通过条件和隐私边界；只有真实 provider、真人、原生权限或真实客户端证据通过 intake 后，才进入 ledger review。

-
awaiting-submission · external

Provider Holdout

No real evidence submission has been provided yet.

负责人
operator with provider credentials
模板
evidence/world_class/templates/provider-holdout.intake.json
提交
evidence/world_class/submissions/provider-holdout.json
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • provider-backed model run
  • observed timing
  • observed token metadata

执行步骤

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

隐私边界

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
awaiting-submission · human

Human Adjudication

No real evidence submission has been provided yet.

负责人
human reviewer
模板
evidence/world_class/templates/human-adjudication.intake.json
提交
evidence/world_class/submissions/human-adjudication.json
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

隐私边界

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.
awaiting-submission · external

Native Permission Enforcement

No real evidence submission has been provided yet.

负责人
target client or installer integrator
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
提交
evidence/world_class/submissions/native-permission-enforcement.json
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

隐私边界

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
awaiting-submission · external

Native Client Telemetry

No real evidence submission has been provided yet.

负责人
Browser/Chrome/IDE/provider client integrator
模板
evidence/world_class/templates/native-client-telemetry.intake.json
提交
evidence/world_class/submissions/native-client-telemetry.json
下一步
Install a real client against the native host and import production metadata-only events.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real external client source
  • metadata-only event
  • local-first import path

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

隐私边界

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
fix-submission · external

Provider Holdout

Submission exists but failed intake validation.

负责人
operator with provider credentials
模板
evidence/world_class/templates/provider-holdout.intake.json
提交
evidence/world_class/submissions/provider-holdout.json
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • provider-backed model run
  • observed timing
  • observed token metadata

执行步骤

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

隐私边界

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
awaiting-submission · human

Human Adjudication

No real evidence submission has been provided yet.

负责人
human reviewer
模板
evidence/world_class/templates/human-adjudication.intake.json
提交
evidence/world_class/submissions/human-adjudication.json
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

隐私边界

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.
awaiting-submission · external

Native Permission Enforcement

No real evidence submission has been provided yet.

负责人
target client or installer integrator
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
提交
evidence/world_class/submissions/native-permission-enforcement.json
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

隐私边界

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
awaiting-submission · external

Native Client Telemetry

No real evidence submission has been provided yet.

负责人
Browser/Chrome/IDE/provider client integrator
模板
evidence/world_class/templates/native-client-telemetry.intake.json
提交
evidence/world_class/submissions/native-client-telemetry.json
下一步
Install a real client against the native host and import production metadata-only events.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real external client source
  • metadata-only event
  • local-first import path

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

隐私边界

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -914,17 +914,17 @@
-

公开声明

本地复现
是
发布锁
否
可公开声明
否
声明阻断
5
Provider 证据
否
人审完成
否
世界级就绪
否
-

声明阻断

  • 阻断release lock is not clean or commit is unavailable
  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (10/19 pass, 9 blocked)
+

公开声明

本地复现
是
发布锁
是
可公开声明
否
声明阻断
3
Provider 证据
是
人审完成
否
世界级就绪
否
+

声明阻断

  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (3 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (11/19 pass, 8 blocked)
-

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true

+

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true

证据台账

Ledger Entry Count
4
Accepted Count
0
待审
4
Human Pending Count
1
External Pending Count
3
Overclaim Guard Active
是
Ready To Claim World Class
否
-

证据入口

Schema
是
模板数
4
模板通过
4
提交数
0
有效提交
0
无效提交
0
可收证据
是
可审台账
否
Ready To Claim World Class
否
+

证据入口

Schema
是
模板数
4
模板通过
4
提交数
1
有效提交
0
无效提交
1
可收证据
否
可审台账
否
Ready To Claim World Class
否

入口边界

intake 只校验证据包格式、来源、隐私和反过度声明；只有 ledger 看到真实 provider、真人、原生权限或真实客户端结果后，world-class 才能进入完成审计。

diff --git a/reports/review-studio.json b/reports/review-studio.json index 1a71c15b..672a0823 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -50,7 +50,7 @@ "key": "output-lab", "label": "输出实验", "status": "warn", - "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5", + "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5", "evidence": "reports/output_quality_scorecard.json", "link": "output_quality_scorecard.md", "review_action_id": "review-action-output-lab", @@ -65,7 +65,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 990/1000; deferred 521572/120000; top deferred scripts 460269; resource governance governed; quality density 131.3", + "detail": "initial load 998/1000; deferred 523653/120000; top deferred scripts 462350; resource governance governed; quality density 140.3", "evidence": "reports/context_budget.json", "link": "context_budget.md", "review_action_id": "", @@ -125,7 +125,7 @@ "key": "architecture-maintainability", "label": "架构维护", "status": "pass", - "detail": "229 Python files; 0 hotspots; 0 watchlist files; 5 early watch files; 0 blockers; largest 706 lines; 68 CLI handlers; 18 in entrypoint", + "detail": "229 Python files; 0 hotspots; 0 watchlist files; 5 early watch files; 0 blockers; largest 711 lines; 68 CLI handlers; 18 in entrypoint", "evidence": "reports/architecture_maintainability.json", "link": "architecture_maintainability.md", "review_action_id": "", @@ -185,7 +185,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false", + "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", "evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", "link": "adoption_drift_report.md", "review_action_id": "", @@ -215,7 +215,7 @@ "key": "world-class-evidence", "label": "世界证据", "status": "warn", - "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true", "evidence": "reports/world_class_evidence_ledger.json", "link": "world_class_evidence_ledger.md", "review_action_id": "review-action-world-class-evidence", @@ -308,7 +308,7 @@ "key": "output-lab", "label": "输出实验", "status": "warn", - "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5", + "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5", "evidence": "reports/output_quality_scorecard.json", "link": "output_quality_scorecard.md", "review_action_id": "review-action-output-lab", @@ -338,7 +338,7 @@ "key": "world-class-evidence", "label": "世界证据", "status": "warn", - "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true", "evidence": "reports/world_class_evidence_ledger.json", "link": "world_class_evidence_ledger.md", "review_action_id": "review-action-world-class-evidence", @@ -652,73 +652,30 @@ "label": "Provider Holdout", "category": "external", "status": "pending", - "readiness": "awaiting-submission", - "current": "model-executed 0; token-observed 0", + "readiness": "fix-submission", + "current": "model-executed 10; token-observed 10", "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", - "source_pass_count": 1, - "source_blocked_count": 2, - "blocked_checks": [ - { - "label": "Provider model run", - "field": "model_executed_count", - "actual": 0, - "expected": ">0", - "status": "blocked", - "next_action": "Run provider-backed output-exec with real credentials." - }, - { - "label": "Token usage observed", - "field": "token_observed_count", - "actual": 0, - "expected": ">0", - "status": "blocked", - "next_action": "Provider execution should return non-estimated token usage." - } - ], + "source_pass_count": 3, + "source_blocked_count": 0, + "blocked_checks": [], "repair_rows": [ { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false } ], - "repair_blocked_count": 3, + "repair_blocked_count": 1, "repair_counts_as_completion": false, "phase_queue": [ { @@ -734,31 +691,13 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false - }, - { - "phase": "collect-source", - "label": "Collect source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false } ], - "phase_queue_blocked_count": 2, + "phase_queue_blocked_count": 1, "phase_queue_counts_as_completion": false, "commands": [ { @@ -788,9 +727,9 @@ } ], "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60" + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120" ], "provenance_requirements": [ "provider-backed model run", @@ -1287,12 +1226,12 @@ "category": "external", "status": "pending", "readiness": "awaiting-submission", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "next_action": "Install a real client against the native host and import production metadata-only events.", "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "blocked_checks": [ { "label": "External events", @@ -1301,6 +1240,14 @@ "expected": ">0", "status": "blocked", "next_action": "Import at least one metadata-only event from a real client." + }, + { + "label": "Adoption sample", + "field": "adoption_sample_count", + "actual": 0, + "expected": ">0", + "status": "blocked", + "next_action": "Telemetry must include adoption outcome evidence." } ], "repair_rows": [ @@ -1317,6 +1264,19 @@ "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "repair_type": "source-check", @@ -1331,7 +1291,7 @@ "counts_as_completion": false } ], - "repair_blocked_count": 2, + "repair_blocked_count": 3, "repair_counts_as_completion": false, "phase_queue": [ { @@ -1357,16 +1317,16 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", - "next_action": "Import at least one metadata-only event from a real client.", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false } @@ -1507,7 +1467,7 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "maturity": "governed", "updated_at": "2026-03-31", "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", @@ -1584,7 +1544,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 360 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -1605,7 +1565,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 360 个词/字，references 约 16934 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -1659,7 +1619,7 @@ }, "contract_boundary": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -1687,7 +1647,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -1709,7 +1669,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -1719,10 +1679,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -1774,7 +1734,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 360 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -1915,7 +1875,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 11, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1929,10 +1889,7 @@ "status": "pending", "summary_zh": "缺少真实 provider 模型运行和 token metadata。", "summary_en": "Missing a real provider model run and token metadata.", - "blocked_checks": [ - "Provider model run", - "Token usage observed" - ] + "blocked_checks": [] }, { "key": "human-adjudication", @@ -1975,7 +1932,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -2139,17 +2097,17 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "logic_steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "usage_steps": [ "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", - "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." ], "package_map": [ { @@ -2216,7 +2174,7 @@ "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." ], "trigger_contract": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -2416,14 +2374,14 @@ "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -2432,9 +2390,19 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "failures": [] }, @@ -2508,43 +2476,43 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d", - "source_contract_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, "command_executed_count": 10, "timing_observed_count": 10, - "model_executed_count": 0, - "token_observed_count": 0, + "model_executed_count": 10, + "token_observed_count": 10, "human_review_complete": false, - "provider_evidence_complete": false, + "provider_evidence_complete": true, "world_class_ready": false, - "world_class_open_gap_count": 4, + "world_class_open_gap_count": 3, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5, + "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 64, - "source_tree_dirty": true, - "source_changed_file_count": 6, + "changed_file_count": 45, + "source_tree_dirty": false, + "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 58 + "generated_changed_file_count": 45 }, - "commit": "ff0473ac02f14a3a431cb01f157453335eb14901", + "commit": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", - "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", "Pending blind-review decisions are visible but do not count as human adjudication.", "World-class readiness remains false until external and human evidence gaps close." ] @@ -2645,7 +2613,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 245, - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" }, "skill_atlas": { "skill_count": 12, @@ -2669,7 +2637,7 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", @@ -2683,7 +2651,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { @@ -2743,7 +2711,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 692, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -2803,7 +2771,7 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ @@ -2815,7 +2783,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "to": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" } ] }, @@ -2832,14 +2800,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -2848,7 +2816,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -2989,15 +2957,15 @@ { "key": "provider-holdout", "label": "Provider Holdout", - "status": "external_required", + "status": "pass", "category": "external", "owner": "operator with provider credentials", - "current": "model-executed 0; token-observed 0", + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -3123,7 +3091,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -3159,27 +3127,27 @@ "source_audit": { "json": "reports/skill_os2_audit.json", "markdown": "reports/skill_os2_audit.md", - "open_gap_count": 4 + "open_gap_count": 3 } }, "world_class_evidence_ledger": { "ok": true, "summary": { "ledger_entry_count": 4, - "source_accepted_count": 0, + "source_accepted_count": 1, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 4, - "invalid_submission_count": 0, + "missing_submission_count": 3, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 11, + "source_blocked_count": 8, "submitted_but_pending_count": 0, - "source_accepted_without_valid_submission_count": 0, + "source_accepted_without_valid_submission_count": 1, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -3191,14 +3159,14 @@ "category": "external", "owner": "operator with provider credentials", "status": "pending", - "source_status": "external_required", - "source_accepted": false, - "current": "model-executed 0; token-observed 0", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -3228,10 +3196,10 @@ "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." ], "observed_state": { - "model_executed_count": 0, + "model_executed_count": 10, "timing_observed_count": 10, - "token_observed_count": 0, - "accepted": false + "token_observed_count": 10, + "accepted": true }, "source_checklist": [ { @@ -3239,9 +3207,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -3251,7 +3219,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -3259,24 +3227,36 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 3, + "source_blocked_count": 0, "submission_state": { - "status": "missing", + "status": "invalid-contract", "path": "evidence/world_class/submissions/provider-holdout.json", - "artifact_ref_count": 0, - "attested_real_evidence": false, - "privacy_contract_satisfied": false, + "submitted_by": "Codex operator using DeepSeek official API", + "submitted_at": "2026-06-17", + "artifact_ref_count": 1, + "artifact_existing_count": 1, + "artifact_sha256_verified_count": 1, + "attested_real_evidence": true, + "reviewer_or_operator_identity_present": true, + "privacy_contract_satisfied": true, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", + "errors": [ + "attestation.ledger_reviewer_approved must be true for a real submission", + "attestation.ledger_reviewer is required", + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + "attestation.ledger_reviewer must be different from submitted_by", + "attestation.ledger_reviewed_at must be at or after submitted_at" + ], "ledger_counts_as_completion": false }, "anti_overclaim": { @@ -3603,7 +3583,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -3640,7 +3620,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -3661,8 +3641,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -3678,8 +3658,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -3715,10 +3695,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -3904,7 +3884,7 @@ "correct_count": 3, "precision": 1.0, "recall": 1.0, - "average_margin": 0.611 + "average_margin": 0.701 }, "team-frontend-review": { "expected_count": 3, @@ -3920,7 +3900,7 @@ "correct_count": 3, "precision": 1.0, "recall": 1.0, - "average_margin": 0.659 + "average_margin": 0.586 }, "no_route": { "expected_count": 4, @@ -3966,17 +3946,16 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.741, + "margin": 1.0, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.741, + "score": 1.0, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" @@ -4007,17 +3986,16 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.535, + "margin": 0.661, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.535, + "score": 0.661, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package" ], "matched_negative_concepts": [] @@ -4046,23 +4024,22 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.558, + "margin": 0.441, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.558, + "score": 0.441, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ - "build_skill", "reuse_package" ], "matched_negative_concepts": [] }, { "name": "governed-incident-command", - "score": 0.269, + "score": 0.181, "passed_threshold": false, "threshold": 0.48, "matched_desired_concepts": [ @@ -4105,7 +4082,7 @@ }, { "name": "yao-meta-skill", - "score": 0.018, + "score": 0.012, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -4145,7 +4122,7 @@ }, { "name": "yao-meta-skill", - "score": 0.013, + "score": 0.008, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -4218,13 +4195,14 @@ "matched_desired_concepts": [ "comms_and_owners", "governed_workflow", + "incident_packet", "severity_assessment" ], "matched_negative_concepts": [] }, { "name": "yao-meta-skill", - "score": 0.021, + "score": 0.016, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -4246,12 +4224,12 @@ "expected_route": "governed-incident-command", "predicted_route": "governed-incident-command", "correct": true, - "margin": 0.805, + "margin": 0.505, "ambiguous": false, "ranked_routes": [ { "name": "governed-incident-command", - "score": 0.805, + "score": 0.505, "passed_threshold": true, "threshold": 0.48, "matched_desired_concepts": [ @@ -4262,7 +4240,7 @@ }, { "name": "yao-meta-skill", - "score": 0.189, + "score": 0.328, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [ @@ -4286,23 +4264,25 @@ "expected_route": "governed-incident-command", "predicted_route": "governed-incident-command", "correct": true, - "margin": 0.172, + "margin": 0.252, "ambiguous": false, "ranked_routes": [ { "name": "governed-incident-command", - "score": 0.62, + "score": 1.0, "passed_threshold": true, "threshold": 0.48, "matched_desired_concepts": [ "comms_and_owners", - "governed_workflow" + "governed_workflow", + "incident_packet", + "severity_assessment" ], "matched_negative_concepts": [] }, { "name": "yao-meta-skill", - "score": 0.448, + "score": 0.748, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ @@ -4372,7 +4352,7 @@ "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.016, + "score": 0.008, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -5301,22 +5281,32 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "summary": { "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -5326,17 +5316,17 @@ "case_id": "skill-package-contract", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 48.44, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 5980.01, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 16, - "output_tokens": 17, - "total_tokens": 33, - "estimated": true + "input_tokens": 100, + "output_tokens": 384, + "total_tokens": 484, + "estimated": false }, "score": 0.0, "passed_count": 0, @@ -5347,47 +5337,49 @@ "has-report", "has-resource-boundary" ], - "output_sha256": "ece26a6edbbbea86a93faaf9f0eadf88efbae12f1c3ad81da9376cf2a889d101", + "output_sha256": "1aa12d65d55863581eef67aaa173b3a9962524a91533693044cb340a342b760b", "failure": "" }, { "case_id": "skill-package-contract", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 50.61, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 11217.19, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 16, - "output_tokens": 57, - "total_tokens": 73, - "estimated": true + "input_tokens": 850, + "output_tokens": 998, + "total_tokens": 1848, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "8bee0131e17fdbe87e48457130ac6aeaa55ea30c919a45e77109852609f33eba", + "score": 75.0, + "passed_count": 3, + "failed_count": 1, + "failed_assertions": [ + "has-report" + ], + "output_sha256": "ac2cb822520b3099150ea1df07de59b9c6df382b7251792e0662631a60989d89", "failure": "" }, { "case_id": "output-eval-expectation", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 51.72, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 3178.95, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 24, - "output_tokens": 12, - "total_tokens": 36, - "estimated": true + "input_tokens": 107, + "output_tokens": 129, + "total_tokens": 236, + "estimated": false }, "score": 0.0, "passed_count": 0, @@ -5398,183 +5390,192 @@ "has-scorecard", "has-failure-taxonomy" ], - "output_sha256": "98fa332fb493b5dc14656a9d65a6c0a9c25ff61a14b71b647aad89cbdbd3ab35", + "output_sha256": "a128ff4dbd7870e287ac651b6cab6b7887e347d6477e5bd8c6f40cd9479539f0", "failure": "" }, { "case_id": "output-eval-expectation", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 39.78, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4611.78, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 24, - "output_tokens": 56, - "total_tokens": 80, - "estimated": true + "input_tokens": 857, + "output_tokens": 264, + "total_tokens": 1121, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "8efdbfefa6a36603682b7063b1753f9f939b3810c082d29686e297906d9f51c8", + "score": 25.0, + "passed_count": 1, + "failed_count": 3, + "failed_assertions": [ + "has-baseline-comparison", + "has-assertion-grading", + "has-failure-taxonomy" + ], + "output_sha256": "4df98d20e2199e56dae2618b6a64e59d5c96484a4f71ae0ebc674b755cf533c3", "failure": "" }, { "case_id": "ir-before-packaging", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 36.93, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 6772.59, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 18, - "output_tokens": 15, - "total_tokens": 33, - "estimated": true + "input_tokens": 104, + "output_tokens": 661, + "total_tokens": 765, + "estimated": false }, - "score": 0.0, - "passed_count": 0, - "failed_count": 4, + "score": 25.0, + "passed_count": 1, + "failed_count": 3, "failed_assertions": [ "has-ir-first", "has-core-ir-fields", - "has-targets", "has-semantic-preservation" ], - "output_sha256": "07a4aa9ee9e48341d4b84d5cb9b6414901aa2c8841340d67447e187240c6c4c2", + "output_sha256": "1696075b4f884ff8b478786b5565aff73ad73235936eb39c2ed615c0746861d2", "failure": "" }, { "case_id": "ir-before-packaging", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 38.12, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 17471.19, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 18, - "output_tokens": 62, - "total_tokens": 80, - "estimated": true + "input_tokens": 854, + "output_tokens": 2132, + "total_tokens": 2986, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "707fde419b3d61e921561b1c4a8523e1f7b53b1acd5758a47ce21d9ed711fde0", - "failure": "" - }, - { - "case_id": "near-neighbor-boundary", - "variant": "baseline", - "status": "pass", - "execution_mode": "command", - "model_executed": false, - "command_executed": true, - "duration_ms": 36.96, - "provider": "local-output-eval-runner", - "model": "", - "usage": { - "input_tokens": 17, - "output_tokens": 19, - "total_tokens": 36, - "estimated": true - }, - "score": 0.0, - "passed_count": 0, + "score": 25.0, + "passed_count": 1, "failed_count": 3, + "failed_assertions": [ + "has-ir-first", + "has-core-ir-fields", + "has-semantic-preservation" + ], + "output_sha256": "4f87633e53f360948d6403126d49ad3dabcb1b47aff596918650719b7840abcf", + "failure": "" + }, + { + "case_id": "near-neighbor-boundary", + "variant": "baseline", + "status": "pass", + "execution_mode": "model", + "model_executed": true, + "command_executed": true, + "duration_ms": 3091.33, + "provider": "deepseek", + "model": "deepseek-v4-flash", + "usage": { + "input_tokens": 105, + "output_tokens": 93, + "total_tokens": 198, + "estimated": false + }, + "score": 33.33, + "passed_count": 1, + "failed_count": 2, "failed_assertions": [ "declines-skill", - "names-near-neighbor", "requires-reuse-signal" ], - "output_sha256": "648ebaa6e09486d05f33fa6639dc973b029d115158588fd192efbe15ccaaeac4", + "output_sha256": "56c8f48f3b2b9963adad3afdb1b6a2bed6b162067506edee1e0f1acd32668a91", "failure": "" }, { "case_id": "near-neighbor-boundary", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 37.81, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4152.85, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 48, - "total_tokens": 65, - "estimated": true + "input_tokens": 855, + "output_tokens": 160, + "total_tokens": 1015, + "estimated": false }, - "score": 100.0, - "passed_count": 3, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "17a3a5286ad573dea86e51f82c04b596babaecb92cde8879f09e7c519205e2e1", + "score": 33.33, + "passed_count": 1, + "failed_count": 2, + "failed_assertions": [ + "declines-skill", + "requires-reuse-signal" + ], + "output_sha256": "d2d85a2aae5db5c4c7e6e0db5bdd865157a90018afe302b0a703940cc84ba68c", "failure": "" }, { "case_id": "file-backed-governed-package", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 37.37, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4750.01, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 20, - "total_tokens": 37, - "estimated": true + "input_tokens": 200, + "output_tokens": 302, + "total_tokens": 502, + "estimated": false }, - "score": 0.0, - "passed_count": 0, - "failed_count": 5, + "score": 60.0, + "passed_count": 3, + "failed_count": 2, "failed_assertions": [ "uses-file-backed-evidence", - "has-governance", - "has-output-contract", - "has-trust-and-scorecard", - "does-not-invent-evidence" + "has-trust-and-scorecard" ], - "output_sha256": "6e5827219459dee34143e4f11663415db80c3111291f75fb6c2115c7bda4cf30", + "output_sha256": "d41880b0c2f7e60ee6acb2cb1f4acb9377fd63de72b8e026e6259a962b89af0b", "failure": "" }, { "case_id": "file-backed-governed-package", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 39.46, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 9406.25, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 81, - "total_tokens": 98, - "estimated": true + "input_tokens": 950, + "output_tokens": 660, + "total_tokens": 1610, + "estimated": false }, - "score": 100.0, - "passed_count": 5, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "bcfdf036666d406abbc945fc42b93eb74d7074118893e540253c7d8715baad73", + "score": 60.0, + "passed_count": 3, + "failed_count": 2, + "failed_assertions": [ + "has-governance", + "has-output-contract" + ], + "output_sha256": "c670ae2ef81a819a9a0f1b0c046e82ba4c649f56dcbe96a45a6938c7f214fba4", "failure": "" } ], @@ -6342,50 +6343,43 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "ff0473ac02f14a3a431cb01f157453335eb14901", + "commit": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "git_status": { "available": true, "dirty": true, - "changed_file_count": 64, + "changed_file_count": 45, "generated_dirty": true, - "generated_changed_file_count": 58, - "source_dirty": true, - "source_changed_file_count": 6, + "generated_changed_file_count": 45, + "source_dirty": false, + "source_changed_file_count": 0, "sample": [ - " M evidence/world_class/README.md", - " M registry/index.json", - " M registry/packages/yao-meta-skill.json", - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/adoption_drift_report.json", - " M reports/architecture_maintainability.json", - " M reports/architecture_maintainability.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", - " M reports/compiled_targets.json", - " M reports/context_budget.json" - ], - "source_sample": [ - " M evidence/world_class/README.md", - " M scripts/evidence_consistency_world_class.py", - " M scripts/render_world_class_operator_runbook.py", - " M tests/verify_evidence_consistency.py", - " M tests/verify_world_class_operator_runbook.py", - "?? scripts/world_class_operator_runbook_coordination.py" - ], - "generated_sample": [ - " M registry/index.json", - " M registry/packages/yao-meta-skill.json", " M reports/adaptation_proposals.json", " M reports/adaptation_proposals.md", " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", " M reports/architecture_maintainability.json", " M reports/architecture_maintainability.md", " M reports/benchmark_reproducibility.json", " M reports/benchmark_reproducibility.md", " M reports/compiled_targets.json", " M reports/context_budget.json", - " M reports/context_budget.md" + " M reports/context_budget.md", + " M reports/context_budget_summary.json" + ], + "source_sample": [], + "generated_sample": [ + " M reports/adaptation_proposals.json", + " M reports/adaptation_proposals.md", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json" ], "generated_dirty_prefixes": [ "dist/", @@ -6399,57 +6393,55 @@ }, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d", - "source_contract_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, "command_executed_count": 10, "timing_observed_count": 10, - "model_executed_count": 0, - "token_observed_count": 0, + "model_executed_count": 10, + "token_observed_count": 10, "human_review_complete": false, - "provider_evidence_complete": false, + "provider_evidence_complete": true, "world_class_ready": false, - "world_class_open_gap_count": 4, + "world_class_open_gap_count": 3, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5, + "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 64, - "source_tree_dirty": true, - "source_changed_file_count": 6, + "changed_file_count": 45, + "source_tree_dirty": false, + "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 58 + "generated_changed_file_count": 45 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ - "release lock is not clean or commit is unavailable", - "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", - "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (10/19 pass, 9 blocked)" + "world-class evidence is not accepted yet (3 open gaps, 4 ledger pending)", + "world-class source checks are not all accepted (11/19 pass, 8 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": false, - "commit": "ff0473ac02f14a3a431cb01f157453335eb14901", + "ready": true, + "commit": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "status_scope": "generation-time status before this report is written", - "source_changed_file_count": 6, - "generated_changed_file_count": 58, - "reason": "source files were dirty at generation time" + "source_changed_file_count": 0, + "generated_changed_file_count": 45, + "reason": "only generated evidence artifacts were dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -6457,7 +6449,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d" + "sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -6530,8 +6522,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7967, - "sha256": "f02f5250076a368521a463325fe1510f2d4334d492464a5270c9f94fab3b9662" + "bytes": 8449, + "sha256": "4df66b63d2e717a559a6881b966fb1b92def7329cc89aa48ee0d8596f87402e5" }, { "label": "blind_review", @@ -6551,8 +6543,8 @@ "label": "trigger_scorecard", "path": "reports/route_scorecard.json", "exists": true, - "bytes": 16961, - "sha256": "c164e83e36d0af276b6af2de2a5e026d7f0711b83eef2b5dcd0e760bd8bb28fc" + "bytes": 16950, + "sha256": "a649547a7d3d5fe02262ca1b188ed4022d8f557770be425f7b7cf762aa76533a" }, { "label": "runtime_conformance", @@ -6565,22 +6557,22 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 137649, - "sha256": "0873eba396f79089e01d15d6e61046e4749a8876bc13c2c604105ea282cbc1e8" + "bytes": 137787, + "sha256": "76e0d31fe1e0e0c6f96eaa2321733ca456d8c59d3eb9007a941bb3d8afdbc02e" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, "bytes": 30127, - "sha256": "2bb4f894fdd29bd3b7714d524e9e9bcee6a4c40d948a34d89a15829acf2be9f7" + "sha256": "d31dc03c74f17242f4163b8cb3f23120fe3d0712edd08fa183cb94f8120d491c" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, - "bytes": 3183, - "sha256": "5132d2d7420f4b0fe09f09fc75756865695a1249564c8ab0356deef8fe7c55ab" + "bytes": 3137, + "sha256": "49e2aee07e38a83608504a3a4f79dc07889e4ba899c4b96859721a5a7cb0a856" }, { "label": "package_verification", @@ -6594,70 +6586,70 @@ "path": "reports/install_simulation.json", "exists": true, "bytes": 8758, - "sha256": "91c2bd418918f2877596e75b986c4ee6c55a50bf00063577be94ae65fc8399f5" + "sha256": "c1f1122264b868c4998482c580ef94333575c6ff7c5c9c297188b225f69a131d" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, - "bytes": 14466, - "sha256": "f68822068601b254ebdf8966c74386ca15845eabcb93b3e82355931ab4651670" + "bytes": 13824, + "sha256": "6362e79a8cfd3ff54b1a7ca32f37de8ed1cda500b953fdf1074928c6d2d62379" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, - "bytes": 22784, - "sha256": "d0ab417742950ff5a5f3407f068829217da5c997a6a7c5a5ee59441cb4abfb7f" + "bytes": 23314, + "sha256": "8eb5e1fbaa7580f107fd9920d13a13b239edb72d50b6959fc8f6dd43150f07d7" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 26016, - "sha256": "fc4c3e398ad0de110ae5f37e461d03712716552ce85b44e41e09f4d343a17f0b" + "bytes": 26940, + "sha256": "a345880034709fcad8741cf20114dba739e2652a76e65d58265f8a87d10c8698" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 20646, - "sha256": "1a939bdd1f4a3ee30e05f0b286145f0f3320ba9f1935749806a581737155774a" + "bytes": 21720, + "sha256": "ebbfa25a5211125583f333fdd41a3dea936a54350cc6361dee751fa1da46881b" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 97462, - "sha256": "d6c13c020987c1b395d8e78cc409fec0784f6746ff77f0ef148abf105b9d6045" + "bytes": 94331, + "sha256": "eecbe8b0eed0581af20140cfdccbf4346b9b0ca8022f4a033940a166c2415195" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 17299, - "sha256": "b256c40e75283556fc2ab1a0ff3a19834e82d54d4c7a3e7464806c01aab69db7" + "bytes": 17725, + "sha256": "56f927077bbdb0f601dc56f0d8645432db8e9a8ad78e323828a91dd2f52a1f1b" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 90823, - "sha256": "f9aa6f5c13af9ee6f5dcb11dafe66b730e5ad41445acf826712b2abef6703a0d" + "bytes": 87920, + "sha256": "94cce92c415b0348348c9b7dbe9eaf233a1b64ff37b5b98e128e11f2f40412ea" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 25405, - "sha256": "f77de0ede45edb4793e4648cdf164bd9caf09e406fd42764f4221a623df175c3" + "bytes": 25549, + "sha256": "94a63a39d2d0989db7a0c6d15f49a5027148654ab30fd197205e7b78173be262" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 34524, - "sha256": "6e07cc2c00a37ae522b7fa1afa09bcb9b6eddf926979582cd2d24fbc9476631f" + "bytes": 34702, + "sha256": "4dc91fdfc2e54a83aedf5ea99e80481949326d74c6933abd9d85c5f4a8f10c5b" }, { "label": "world_class_claim_guard", @@ -6792,7 +6784,7 @@ }, "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", - "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", "Pending blind-review decisions are visible but do not count as human adjudication.", "World-class readiness remains false until external and human evidence gaps close." ], @@ -7615,7 +7607,7 @@ "compiled_targets": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "target_count": 5, @@ -7652,10 +7644,10 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -7691,22 +7683,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", - "`Governed`: high-trust, policy-sensitive, or release-critical.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -8237,7 +8225,7 @@ "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", "activation": { "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -8493,7 +8481,7 @@ "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", "activation": { "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -8607,10 +8595,10 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -8646,22 +8634,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", - "`Governed`: high-trust, policy-sensitive, or release-critical.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -9192,7 +9176,7 @@ "native_surface": "Claude-compatible neutral source folder with adapter notes", "activation": { "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -9448,7 +9432,7 @@ "native_surface": "Claude-compatible neutral source folder with adapter notes", "activation": { "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -9562,10 +9546,10 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -9601,22 +9585,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", - "`Governed`: high-trust, policy-sensitive, or release-critical.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -10147,7 +10127,7 @@ "native_surface": "Agent Skills compatible neutral package", "activation": { "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -10396,7 +10376,7 @@ "native_surface": "Agent Skills compatible neutral package", "activation": { "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -10501,10 +10481,10 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -10540,22 +10520,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", - "`Governed`: high-trust, policy-sensitive, or release-critical.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -11086,7 +11062,7 @@ "native_surface": "Agent Skills standard source tree", "activation": { "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -11335,7 +11311,7 @@ "native_surface": "Agent Skills standard source tree", "activation": { "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -11440,10 +11416,10 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -11479,22 +11455,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", - "`Governed`: high-trust, policy-sensitive, or release-critical.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -12025,7 +11997,7 @@ "native_surface": "VS Code/Copilot Agent Skills project or user scope", "activation": { "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -12278,7 +12250,7 @@ "native_surface": "VS Code/Copilot Agent Skills project or user scope", "activation": { "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, @@ -13026,7 +12998,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 245, - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" }, "failures": [], "warnings": [], @@ -13669,9 +13641,11 @@ "uses_file_write": false, "uses_subprocess": false, "network_urls": [ - "https://api.openai.com/v1/responses" + "https://api.openai.com/v1/responses", + "https://api.deepseek.com/chat/completions" ], "network_hosts": [ + "api.deepseek.com", "api.openai.com" ] }, @@ -15516,7 +15490,7 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: provider_output_eval_runner.py [-h] [--provider PROVIDER]\n [--base-url BASE_URL] [--model MODEL]\n [--api-key-env API_KEY_ENV]\n ", + "stdout_excerpt": "usage: provider_output_eval_runner.py [-h] [--provider PROVIDER]\n [--base-url BASE_URL]\n [--api-format {responses,chat-completions}]\n ", "stderr_excerpt": "" }, { @@ -16395,7 +16369,7 @@ ], "reviewer": "Yao Team", "scope": "Only scripts declared in security/network_policy.json may perform outbound requests.", - "reason": "Network use is bounded to public GitHub metadata and benchmark discovery, with allowed hosts reviewed in security/network_policy.json.", + "reason": "Network use is bounded to public GitHub metadata, benchmark discovery, update checks, and reviewed provider output-eval endpoints, with allowed hosts reviewed in security/network_policy.json.", "expires_at": "2026-09-30", "evidence": [ "security/network_policy.json", @@ -16552,7 +16526,7 @@ "python_compatibility": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "root": ".", "summary": { "target_python": "3.11", @@ -17975,7 +17949,7 @@ "architecture_maintainability": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "python_file_count": 229, @@ -17990,7 +17964,7 @@ "watch_line_threshold": 720, "early_watch_line_threshold": 600, "block_line_threshold": 1500, - "largest_file_lines": 706, + "largest_file_lines": 711, "watchlist_count": 0, "early_watchlist_count": 5, "hotspot_count": 0, @@ -18000,15 +17974,7 @@ "largest_files": [ { "path": "tests/verify_evidence_consistency.py", - "lines": 706, - "kind": "test", - "severity": "pass", - "early_watch": true, - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, - { - "path": "tests/verify_yao_cli.py", - "lines": 696, + "lines": 711, "kind": "test", "severity": "pass", "early_watch": true, @@ -18016,7 +17982,15 @@ }, { "path": "tests/verify_world_class_evidence_intake.py", - "lines": 690, + "lines": 703, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_yao_cli.py", + "lines": 700, "kind": "test", "severity": "pass", "early_watch": true, @@ -18032,7 +18006,7 @@ }, { "path": "scripts/render_world_class_operator_runbook.py", - "lines": 647, + "lines": 651, "kind": "cli-script", "severity": "pass", "early_watch": true, @@ -18054,6 +18028,14 @@ "early_watch": false, "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "tests/verify_world_class_evidence_ledger.py", + "lines": 587, + "kind": "test", + "severity": "pass", + "early_watch": false, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/build_skill_atlas.py", "lines": 586, @@ -18078,14 +18060,6 @@ "early_watch": false, "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "tests/verify_world_class_evidence_ledger.py", - "lines": 579, - "kind": "test", - "severity": "pass", - "early_watch": false, - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "scripts/render_review_studio.py", "lines": 578, @@ -18099,15 +18073,7 @@ "early_watchlist": [ { "path": "tests/verify_evidence_consistency.py", - "lines": 706, - "kind": "test", - "severity": "pass", - "early_watch": true, - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, - { - "path": "tests/verify_yao_cli.py", - "lines": 696, + "lines": 711, "kind": "test", "severity": "pass", "early_watch": true, @@ -18115,7 +18081,15 @@ }, { "path": "tests/verify_world_class_evidence_intake.py", - "lines": 690, + "lines": 703, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_yao_cli.py", + "lines": 700, "kind": "test", "severity": "pass", "early_watch": true, @@ -18131,7 +18105,7 @@ }, { "path": "scripts/render_world_class_operator_runbook.py", - "lines": 647, + "lines": 651, "kind": "cli-script", "severity": "pass", "early_watch": true, @@ -18152,16 +18126,16 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 797, - "other_text_tokens": 1151006, - "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1151803, - "deferred_resource_tokens": 521572, + "skill_body_tokens": 805, + "other_text_tokens": 1152733, + "estimated_initial_load_tokens": 998, + "estimated_total_text_tokens": 1153538, + "deferred_resource_tokens": 523653, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 460269, + "estimated_tokens": 462350, "file_count": 153 }, { @@ -18183,7 +18157,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 460269, + "estimated_tokens": 462350, "file_count": 153 } ], @@ -18206,7 +18180,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 460269, + "estimated_tokens": 462350, "file_count": 153, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } @@ -18215,8 +18189,8 @@ }, "relevant_file_count": 712, "unused_resource_dirs": [], - "quality_signal_points": 130, - "quality_density": 131.3 + "quality_signal_points": 140, + "quality_density": 140.3 } }, "promotion": { @@ -18230,9 +18204,9 @@ { "name": "yao-meta-skill", "winner_label": "Current", - "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "winner_tokens": 65, - "current_tokens": 65, + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "winner_tokens": 53, + "current_tokens": 53, "winner_holdout_fp": 0, "winner_holdout_fn": 0, "current_holdout_fp": 0, @@ -18263,44 +18237,44 @@ "calibration": { "holdout": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "blind_holdout": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "adversarial_holdout": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -18391,7 +18365,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.611, + "average_margin": 0.701, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -18414,8 +18388,8 @@ "label": "Current", "ranking_state": "selected_by_dev", "promotion_state": "kept_current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "tokens": 65, + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "reason_tags": [] @@ -18445,7 +18419,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -18459,7 +18434,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -18759,13 +18735,13 @@ "current_holdout_fp": 0, "current_holdout_fn": 1, "baseline_holdout_fp": 0, - "baseline_holdout_fn": 2, + "baseline_holdout_fn": 1, "winner_blind_holdout_fp": 0, "winner_blind_holdout_fn": 0, "current_blind_holdout_fp": 0, "current_blind_holdout_fn": 0, "baseline_blind_holdout_fp": 0, - "baseline_blind_holdout_fn": 1, + "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_fp": 0, "winner_judge_blind_holdout_fn": 0, @@ -18784,7 +18760,7 @@ "calibration": { "holdout": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -18799,49 +18775,49 @@ }, "blind_holdout": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "adversarial_holdout": { "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" } }, "judge_blind": { "winner": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "current": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "baseline": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -18904,7 +18880,8 @@ "causes": [ "no_candidate_outperformed_current", "current_holdout_gap_present", - "current_holdout_risk" + "current_holdout_risk", + "adversarial_watch_risk" ], "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", "gate_status": { @@ -18920,7 +18897,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.659, + "average_margin": 0.586, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -18934,8 +18911,8 @@ "promotion_state": "reference", "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2 + "dev_errors": 0, + "holdout_errors": 1 }, { "name": "governed-incident-command", @@ -18945,25 +18922,10 @@ "promotion_state": "kept_current", "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [] }, - { - "name": "governed-incident-command", - "role": "candidate", - "label": "Guardrail", - "ranking_state": "not_selected", - "promotion_state": "blocked", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", - "tokens": 51, - "dev_errors": 1, - "holdout_errors": 2, - "reason_tags": [ - "visible_holdout_regression", - "longer_without_gain" - ] - }, { "name": "governed-incident-command", "role": "candidate", @@ -18972,7 +18934,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 54, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -18986,7 +18948,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", "tokens": 78, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -19000,9 +18962,24 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 78, + "dev_errors": 0, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, "dev_errors": 1, "holdout_errors": 1, "reason_tags": [ + "weaker_dev_fit", "longer_without_gain" ] } @@ -19062,7 +19039,7 @@ { "name": "yao-meta-skill", "path": ".", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "owner": "Yao Team", "version": "1.1.0", "status": "active", @@ -19273,23 +19250,21 @@ "templates/complex_skill.md.j2" ], "token_set": [ - "add", - "asked", + "creation", + "distribution", "docs", "evals", "evaluate", - "existing", - "improve", + "improvement", "notes", "or", "package", - "process", + "packaging", "prompts", "refactor", - "repeated", - "reuse", - "team", + "team-ready", "transcripts", + "workflow", "workflows" ], "atlas_scope": "release", @@ -20132,7 +20107,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-17T04:05:21Z", + "generated_at": "2026-06-17T07:53:09Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -20156,14 +20131,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -20172,7 +20147,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -20184,31 +20159,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "skill_activation", + "event": "review_event", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } @@ -20216,7 +20191,7 @@ "daily_skillops": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T04:04:43Z", + "generated_at": "2026-06-17T07:53:10Z", "skill_dir": ".", "decision": "proposal-review", "source_supplied": false, @@ -20229,7 +20204,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -20246,7 +20221,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -20695,7 +20670,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true }, "actions": [ @@ -20730,7 +20705,7 @@ "weekly_curator": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T04:04:43Z", + "generated_at": "2026-06-17T07:53:10Z", "skill_dir": ".", "decision": "curator-review", "week_id": "2026-W25", @@ -20744,7 +20719,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -20764,7 +20739,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -21017,7 +20992,7 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4 @@ -21052,7 +21027,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "summary": { "waiver_count": 0, "active_count": 0, @@ -21135,14 +21110,14 @@ "false-positive", "temporary-exception" ], - "risk_summary": "review pending 5; model-executed 0; output failures 0", + "risk_summary": "review pending 5; model-executed 10; output failures 0", "required_review": [ "Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.", "Reviewer names the release scope and expiry date.", "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { @@ -21172,7 +21147,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -21195,7 +21170,7 @@ "adaptation_proposals": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T04:04:43Z", + "generated_at": "2026-06-17T07:53:10Z", "skill_dir": ".", "source_patterns": "reports/user_patterns.json", "pattern_count": 5, @@ -21598,37 +21573,37 @@ "decision": "evidence-pending", "ready_to_claim_world_class": false, "ledger_entry_count": 4, - "source_accepted_count": 0, + "source_accepted_count": 1, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 4, - "invalid_submission_count": 0, + "missing_submission_count": 3, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 11, + "source_blocked_count": 8, "submitted_but_pending_count": 0, - "source_accepted_without_valid_submission_count": 0, + "source_accepted_without_valid_submission_count": 1, "overclaim_guard_active": true, "summary": { "ledger_entry_count": 4, - "source_accepted_count": 0, + "source_accepted_count": 1, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 4, - "invalid_submission_count": 0, + "missing_submission_count": 3, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 11, + "source_blocked_count": 8, "submitted_but_pending_count": 0, - "source_accepted_without_valid_submission_count": 0, + "source_accepted_without_valid_submission_count": 1, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -21666,14 +21641,14 @@ "category": "external", "owner": "operator with provider credentials", "status": "pending", - "source_status": "external_required", - "source_accepted": false, - "current": "model-executed 0; token-observed 0", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -21703,10 +21678,10 @@ "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." ], "observed_state": { - "model_executed_count": 0, + "model_executed_count": 10, "timing_observed_count": 10, - "token_observed_count": 0, - "accepted": false + "token_observed_count": 10, + "accepted": true }, "source_checklist": [ { @@ -21714,9 +21689,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -21726,7 +21701,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -21734,24 +21709,36 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 3, + "source_blocked_count": 0, "submission_state": { - "status": "missing", + "status": "invalid-contract", "path": "evidence/world_class/submissions/provider-holdout.json", - "artifact_ref_count": 0, - "attested_real_evidence": false, - "privacy_contract_satisfied": false, + "submitted_by": "Codex operator using DeepSeek official API", + "submitted_at": "2026-06-17", + "artifact_ref_count": 1, + "artifact_existing_count": 1, + "artifact_sha256_verified_count": 1, + "attested_real_evidence": true, + "reviewer_or_operator_identity_present": true, + "privacy_contract_satisfied": true, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", + "errors": [ + "attestation.ledger_reviewer_approved must be true for a real submission", + "attestation.ledger_reviewer is required", + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + "attestation.ledger_reviewer must be different from submitted_by", + "attestation.ledger_reviewed_at must be at or after submitted_at" + ], "ledger_counts_as_completion": false }, "anti_overclaim": { @@ -22078,7 +22065,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -22115,7 +22102,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -22136,8 +22123,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -22153,8 +22140,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -22194,7 +22181,7 @@ }, "world_class_evidence_intake": { "schema_version": "1.0", - "ok": true, + "ok": false, "generated_at": "2026-06-17", "skill_dir": ".", "summary": { @@ -22202,17 +22189,17 @@ "ledger_entry_count": 4, "template_count": 4, "template_pass_count": 4, - "submission_count": 0, + "submission_count": 1, "valid_submission_count": 0, - "invalid_submission_count": 0, + "invalid_submission_count": 1, "operator_checklist_count": 4, "operator_checklist_ready_count": 0, "valid_packet_source_incomplete_count": 0, - "ready_for_external_collection": true, + "ready_for_external_collection": false, "ready_for_ledger_review": false, "ready_to_claim_world_class": false, "overclaim_guard_active": true, - "decision": "awaiting-submissions" + "decision": "fix-intake" }, "schema": "evidence/world_class/intake.schema.json", "templates": [ @@ -22273,23 +22260,44 @@ "errors": [] } ], - "submissions": [], + "submissions": [ + { + "path": "evidence/world_class/submissions/provider-holdout.json", + "evidence_key": "provider-holdout", + "status": "fail", + "template_only": false, + "artifact_integrity": { + "artifact_ref_count": 1, + "artifact_existing_count": 1, + "artifact_sha256_verified_count": 1, + "required_artifact_count": 1, + "required_artifact_verified_count": 1 + }, + "errors": [ + "attestation.ledger_reviewer_approved must be true for a real submission", + "attestation.ledger_reviewer is required", + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + "attestation.ledger_reviewer must be different from submitted_by", + "attestation.ledger_reviewed_at must be at or after submitted_at" + ] + } + ], "operator_checklist": [ { "evidence_key": "provider-holdout", "label": "Provider Holdout", "category": "external", "owner": "operator with provider credentials", - "readiness": "awaiting-submission", - "blocking_reason": "No real evidence submission has been provided yet.", + "readiness": "fix-submission", + "blocking_reason": "Submission exists but failed intake validation.", "template_status": "pass", - "submission_status": "missing", - "source_accepted": false, + "submission_status": "fail", + "source_accepted": true, "observed_state": { - "model_executed_count": 0, + "model_executed_count": 10, "timing_observed_count": 10, - "token_observed_count": 0, - "accepted": false + "token_observed_count": 10, + "accepted": true }, "template_path": "evidence/world_class/templates/provider-holdout.intake.json", "submission_path": "evidence/world_class/submissions/provider-holdout.json", @@ -22307,9 +22315,9 @@ "observed token metadata" ], "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -22522,7 +22530,7 @@ "source_accepted": false, "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -22606,14 +22614,14 @@ "collection_ready_count": 1, "collection_blocked_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, - "repair_checklist_count": 13, - "repair_blocked_count": 13, + "source_pass_count": 11, + "source_blocked_count": 8, + "repair_checklist_count": 12, + "repair_blocked_count": 12, "repair_ready_count": 0, "repair_phase_counts": { "unblock-access": 4, - "collect-source": 9 + "collect-source": 8 }, "next_repair_action_id": "human-adjudication-precheck-human-reviewer", "next_repair_phase": "unblock-access", @@ -22622,7 +22630,7 @@ "repair_counts_as_completion": false, "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 13, + "phase_queue_row_count": 12, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -22652,9 +22660,9 @@ "owner": "operator with provider credentials", "status": "blocked", "ledger_status": "pending", - "intake_readiness": "awaiting-submission", - "review_state": "awaiting-submission", - "source_accepted": false, + "intake_readiness": "fix-submission", + "review_state": "fix-submission", + "source_accepted": true, "prechecks": [ { "evidence_key": "provider-holdout", @@ -22682,16 +22690,20 @@ }, { "evidence_key": "provider-holdout", - "key": "openai-api-key", + "key": "provider-api-key", "label": "Provider credential", - "kind": "env", + "kind": "env_any", "required": true, - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "secret_value_redacted": true, - "env": "OPENAI_API_KEY", + "env_any": [ + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY" + ], "status": "missing", "actual": "not-set", - "default": "" + "set_count": 0, + "set_envs": [] }, { "evidence_key": "provider-holdout", @@ -22699,12 +22711,12 @@ "label": "Provider model", "kind": "env", "required": false, - "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", "secret_value_redacted": false, "env": "YAO_OUTPUT_EVAL_MODEL", "status": "optional", "actual": "not-set", - "default": "gpt-4.1-mini" + "default": "provider-specific model, or pass --provider-model" } ], "source_checklist": [ @@ -22713,9 +22725,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -22725,7 +22737,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -22733,13 +22745,13 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", "commands": { @@ -22784,58 +22796,28 @@ "artifact_prefill_counts_as_evidence": false }, "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" ], "repair_checklist": [ { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ], "phase_queue": [ @@ -22852,77 +22834,27 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false } ] - }, - { - "phase": "collect-source", - "label": "Collect source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false, - "rows": [ - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - } - ] } ] }, @@ -23668,8 +23600,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -23753,6 +23685,21 @@ "source": "prechecks", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -23810,19 +23757,34 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", - "next_action": "Import at least one metadata-only event from a real client.", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -23870,16 +23832,20 @@ }, { "evidence_key": "provider-holdout", - "key": "openai-api-key", + "key": "provider-api-key", "label": "Provider credential", - "kind": "env", + "kind": "env_any", "required": true, - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "secret_value_redacted": true, - "env": "OPENAI_API_KEY", + "env_any": [ + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY" + ], "status": "missing", "actual": "not-set", - "default": "" + "set_count": 0, + "set_envs": [] }, { "evidence_key": "provider-holdout", @@ -23887,12 +23853,12 @@ "label": "Provider model", "kind": "env", "required": false, - "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", "secret_value_redacted": false, "env": "YAO_OUTPUT_EVAL_MODEL", "status": "optional", "actual": "not-set", - "default": "gpt-4.1-mini" + "default": "provider-specific model, or pass --provider-model" }, { "evidence_key": "human-adjudication", @@ -24018,9 +23984,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -24030,7 +23996,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -24038,9 +24004,9 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." }, { @@ -24188,8 +24154,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -24251,16 +24217,16 @@ "counts_as_completion": false }, { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false @@ -24340,6 +24306,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -24369,36 +24350,6 @@ "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "source_checklist", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ], "phase_queue": [ @@ -24472,16 +24423,16 @@ "counts_as_completion": false }, { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false @@ -24493,19 +24444,17 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 9, - "row_count": 9, + "blocked_count": 8, + "row_count": 8, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", - "operator with provider credentials", "target client or installer integrator" ], "evidence_keys": [ "human-adjudication", "native-client-telemetry", - "native-permission-enforcement", - "provider-holdout" + "native-permission-enforcement" ], "next_action_id": "human-adjudication-source-check-blind_review_attested", "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", @@ -24587,6 +24536,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -24616,36 +24580,6 @@ "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "source_checklist", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ] } @@ -24704,38 +24638,38 @@ }, "world_class_submission_review": { "schema_version": "1.0", - "ok": true, + "ok": false, "generated_at": "2026-06-17", "skill_dir": ".", - "decision": "awaiting-submissions", + "decision": "fix-submissions", "ready_to_claim_world_class": false, "review_item_count": 4, "accepted_count": 0, - "awaiting_submission_count": 4, + "awaiting_submission_count": 3, "valid_packet_source_incomplete_count": 0, "ready_for_ledger_review_count": 0, - "fix_submission_count": 0, + "fix_submission_count": 1, "unmatched_submission_count": 0, - "invalid_submission_count": 0, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 11, + "source_blocked_count": 8, "review_counts_submission_as_completion": false, "summary": { "review_item_count": 4, "accepted_count": 0, - "awaiting_submission_count": 4, + "awaiting_submission_count": 3, "valid_packet_source_incomplete_count": 0, "ready_for_ledger_review_count": 0, - "fix_submission_count": 0, + "fix_submission_count": 1, "unmatched_submission_count": 0, - "invalid_submission_count": 0, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 11, + "source_blocked_count": 8, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, - "decision": "awaiting-submissions" + "decision": "fix-submissions" }, "report_contract": { "schema_version": "1.0", @@ -24770,21 +24704,27 @@ "category": "external", "owner": "operator with provider credentials", "ledger_status": "pending", - "submission_status": "missing", - "intake_status": "missing", - "source_accepted": false, - "review_state": "awaiting-submission", - "blocking_reason": "No evidence packet has been submitted for review.", + "submission_status": "invalid-contract", + "intake_status": "fail", + "source_accepted": true, + "review_state": "fix-submission", + "blocking_reason": "Submission exists but fails the ledger submission contract.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "", - "submitted_at": "", - "artifact_ref_count": 0, - "intake_errors": [], + "submitted_by": "Codex operator using DeepSeek official API", + "submitted_at": "2026-06-17", + "artifact_ref_count": 1, + "intake_errors": [ + "attestation.ledger_reviewer_approved must be true for a real submission", + "attestation.ledger_reviewer is required", + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + "attestation.ledger_reviewer must be different from submitted_by", + "attestation.ledger_reviewed_at must be at or after submitted_at" + ], "observed_state": { - "model_executed_count": 0, + "model_executed_count": 10, "timing_observed_count": 10, - "token_observed_count": 0, - "accepted": false + "token_observed_count": 10, + "accepted": true }, "source_checklist": [ { @@ -24792,9 +24732,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -24804,7 +24744,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -24812,15 +24752,15 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 3, + "source_blocked_count": 0, "success_checks": [ "reports/output_execution_runs.json summary.model_executed_count > 0", "reports/output_execution_runs.json summary.timing_observed_count > 0", @@ -25082,7 +25022,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -25103,8 +25043,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -25120,8 +25060,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", @@ -25153,18 +25093,18 @@ "evidence_item_count": 4, "pending_count": 4, "accepted_count": 0, - "awaiting_submission_count": 4, + "awaiting_submission_count": 3, "ready_for_ledger_review_count": 0, "valid_packet_source_incomplete_count": 0, - "invalid_submission_count": 0, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, - "repair_checklist_count": 13, - "repair_blocked_count": 13, + "source_pass_count": 11, + "source_blocked_count": 8, + "repair_checklist_count": 12, + "repair_blocked_count": 12, "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 13, + "phase_queue_row_count": 12, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -25197,20 +25137,20 @@ "category": "external", "owner": "operator with provider credentials", "ledger_status": "pending", - "intake_readiness": "awaiting-submission", - "review_state": "awaiting-submission", - "source_accepted": false, + "intake_readiness": "fix-submission", + "review_state": "fix-submission", + "source_accepted": true, "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", - "current": "model-executed 0; token-observed 0", + "current": "model-executed 10; token-observed 10", "execution_runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" ], - "blocking_reason": "No evidence packet has been submitted for review.", + "blocking_reason": "Submission exists but fails the ledger submission contract.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", "commands": { @@ -25247,10 +25187,10 @@ "reports/world_class_evidence_intake.md" ], "observed_state": { - "model_executed_count": 0, + "model_executed_count": 10, "timing_observed_count": 10, - "token_observed_count": 0, - "accepted": false + "token_observed_count": 10, + "accepted": true }, "source_checklist": [ { @@ -25258,9 +25198,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -25270,7 +25210,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -25278,65 +25218,32 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], - "blocked_source_check_count": 2, - "next_source_actions": [ - "Run provider-backed output-exec with real credentials.", - "Provider execution should return non-estimated token usage." - ], + "blocked_source_check_count": 0, + "next_source_actions": [], "repair_checklist": [ { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ], - "repair_blocked_count": 3, + "repair_blocked_count": 1, "repair_counts_as_completion": false, "phase_queue": [ { @@ -25352,90 +25259,52 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false } ] - }, - { - "phase": "collect-source", - "label": "Collect source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false, - "rows": [ - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - } - ] } ], - "phase_queue_blocked_count": 2, + "phase_queue_blocked_count": 1, "phase_queue_counts_as_completion": false, "submission_state": { - "status": "missing", + "status": "invalid-contract", "path": "evidence/world_class/submissions/provider-holdout.json", - "artifact_ref_count": 0, - "attested_real_evidence": false, - "privacy_contract_satisfied": false, + "submitted_by": "Codex operator using DeepSeek official API", + "submitted_at": "2026-06-17", + "artifact_ref_count": 1, + "artifact_existing_count": 1, + "artifact_sha256_verified_count": 1, + "attested_real_evidence": true, + "reviewer_or_operator_identity_present": true, + "privacy_contract_satisfied": true, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", + "errors": [ + "attestation.ledger_reviewer_approved must be true for a real submission", + "attestation.ledger_reviewer is required", + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + "attestation.ledger_reviewer must be different from submitted_by", + "attestation.ledger_reviewed_at must be at or after submitted_at" + ], "ledger_counts_as_completion": false }, "anti_overclaim": { @@ -26129,7 +25998,7 @@ "review_state": "awaiting-submission", "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "execution_runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -26177,7 +26046,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -26198,8 +26067,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -26214,9 +26083,10 @@ "next_action": "Telemetry must stay metadata-only." } ], - "blocked_source_check_count": 1, + "blocked_source_check_count": 2, "next_source_actions": [ - "Import at least one metadata-only event from a real client." + "Import at least one metadata-only event from a real client.", + "Telemetry must include adoption outcome evidence." ], "repair_checklist": [ { @@ -26234,6 +26104,21 @@ "source": "prechecks", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -26250,7 +26135,7 @@ "counts_as_completion": false } ], - "repair_blocked_count": 2, + "repair_blocked_count": 3, "repair_counts_as_completion": false, "phase_queue": [ { @@ -26293,19 +26178,34 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", - "next_action": "Import at least one metadata-only event from a real client.", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -26367,10 +26267,10 @@ "evidence_key": "provider-holdout", "owner": "assistant + operator with provider credentials", "requires_user_input": true, - "user_action": "Provide the provider API key through an environment variable and confirm the model to use.", + "user_action": "Provide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use.", "assistant_action": "Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet.", "external_dependency": "Valid provider credentials and a live provider endpoint.", - "command": "python3 scripts/yao.py output-exec . --provider-runner openai --provider-model --timeout-seconds 60", + "command": "python3 scripts/yao.py output-exec . --provider-runner --provider-model --timeout-seconds 60", "pass_condition": "reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0.", "artifact": "reports/output_execution_runs.json", "privacy_boundary": "Commit aggregate metadata only; do not commit API keys, raw prompts, raw outputs, or provider request payloads.", @@ -26568,16 +26468,16 @@ "counts_as_completion": false }, { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false @@ -26657,6 +26557,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -26686,36 +26601,6 @@ "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "source_checklist", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ], "phase_queue": [ @@ -26789,16 +26674,16 @@ "counts_as_completion": false }, { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false @@ -26810,19 +26695,17 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 9, - "row_count": 9, + "blocked_count": 8, + "row_count": 8, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", - "operator with provider credentials", "target client or installer integrator" ], "evidence_keys": [ "human-adjudication", "native-client-telemetry", - "native-permission-enforcement", - "provider-holdout" + "native-permission-enforcement" ], "next_action_id": "human-adjudication-source-check-blind_review_attested", "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", @@ -26904,6 +26787,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -26933,36 +26831,6 @@ "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "source_checklist", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ] } @@ -27772,7 +27640,7 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", @@ -27786,7 +27654,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { @@ -27843,7 +27711,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" } ] }, @@ -28532,7 +28400,7 @@ "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 692, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -28867,7 +28735,7 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ @@ -28879,7 +28747,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "to": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" } ] }, @@ -28924,10 +28792,9 @@ }, "frontmatter": { "name": "yao-meta-skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "metadata": { - "author": "Yao Team", - "philosophy": "structured design, evaluation loop, template ergonomics, operational packaging" + "author": "Yao Team" } }, "interface": { diff --git a/reports/review-viewer.html b/reports/review-viewer.html index 4ab067f7..dc661185 100644 --- a/reports/review-viewer.html +++ b/reports/review-viewer.html @@ -188,7 +188,7 @@ th {

Review Viewer

Yao Meta Skill

-

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

maturity: governed archetype: governed @@ -200,17 +200,17 @@ th {

Architecture at a glance

-
Inputs
workflow, prompt, transcript, docs, or notes
Boundary
Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
Logic
Decide whether the request should become a skill and choose the lightest fit.; Capture job, output, exclusions, constraints, and standards.; Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
Usage
Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.; Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
Next
Review the top iteration directions before growing the package.
+
Inputs
workflow, prompt, transcript, docs, or notes
Boundary
Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
Logic
For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.; Capture job, output, exclusions, constraints, standards, and the lightest fit.; Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.
Usage
Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.; Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
Next
Review the top iteration directions before growing the package.

Core logic

-
  • Decide whether the request should become a skill and choose the lightest fit.
  • Capture job, output, exclusions, constraints, and standards.
  • Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
  • Write the `description` early and test route quality before expanding the package.
  • Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.
+
  • For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.
  • Capture job, output, exclusions, constraints, standards, and the lightest fit.
  • Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.
  • Write `description` early, test route quality, then add only earned folders and gates.
  • Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful.

How to use it

-
  • Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.
  • Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
+
  • Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.
  • Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
@@ -232,20 +232,20 @@ th {

Compare view

-

Winner: Current

{'priority': ['fewest false positives', 'fewest false negatives', 'highest near-neighbor pass rate', 'highest negative pass rate', 'highest precision', 'highest recall', 'shortest description']}

VariantTokensDev ErrorsHoldout ErrorsStrategy
Baseline800existing
Current6500current
Current6500current
+

Winner: Current

{'priority': ['fewest false positives', 'fewest false negatives', 'highest near-neighbor pass rate', 'highest negative pass rate', 'highest precision', 'highest recall', 'shortest description']}

VariantTokensDev ErrorsHoldout ErrorsStrategy
Baseline800existing
Current5300current
Current5300current

Variant diff studio

-

Baseline

existing

Create and improve agent skills.

tokens 8 (0)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create and improve agent skills.

Drops from baseline

No dropped cues.

Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

tokens 65 (+57)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

Drops from baseline

  • Create and improve agent skills.

Winner — Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

tokens 65 (+57)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

Drops from baseline

  • Create and improve agent skills.
+

Baseline

existing

Create and improve agent skills.

tokens 8 (0)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create and improve agent skills.

Drops from baseline

No dropped cues.

Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

tokens 53 (+45)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

Drops from baseline

  • Create and improve agent skills.

Winner — Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

tokens 53 (+45)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

Drops from baseline

  • Create and improve agent skills.

Evidence readiness

Readiness score: 85/100

-
  • Intent clarity · ready
    100/100 intent confidence.
  • Benchmark coverage · ready
    3 GitHub benchmark repositories attached.
  • Pattern gate · ready
    5 accepted, 1 deferred using threshold 4/4.
  • Conflict handling · decision needed
    The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.
  • Output risk profile · ready
    5 output risk families attached.
  • Artifact design profile · ready
    Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.
  • Prompt quality profile · ready
    89.0/100 prompt-facing quality score.
+
  • Intent clarity · ready
    100/100 intent confidence.
  • Benchmark coverage · ready
    3 GitHub benchmark repositories attached.
  • Pattern gate · ready
    5 accepted, 1 deferred using threshold 4/4.
  • Conflict handling · decision needed
    The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.
  • Output risk profile · ready
    5 output risk families attached.
  • Artifact design profile · ready
    Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.
  • Prompt quality profile · ready
    89.0/100 prompt-facing quality score.

Honest boundary check

@@ -272,11 +272,11 @@ th {

Artifact design profile

Design system: metric editorial

-
  • Code, CLI, or implementation guide
    Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.
  • Report or brief
    High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
  • Review viewer
    Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix.
+
  • Dashboard or metrics page
    Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.
  • Review viewer
    Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix.
  • Code, CLI, or implementation guide
    Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.

Visual quality gates

-
  • Name the working directory and required inputs for commands.
  • Mark destructive, networked, or external side-effect operations.
  • Prefer the smallest runnable snippet over broad framework scaffolding.
  • Keep the first screen useful without requiring the reader to parse every detail.
  • Use tables only for comparisons; move explanations below the table.
+
  • Avoid paragraph-heavy table cells.
  • Keep charts tied to one analytical question each.
  • Preserve stable color meaning across metrics and entities.
  • Make differences visible instead of hiding them in prose.
  • Separate author-facing recommendations from reviewer-only evidence.
diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 2c8d7dae..44bd8e8a 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -4,7 +4,7 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "maturity": "governed", "updated_at": "2026-03-31", "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", @@ -81,7 +81,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 360 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -102,7 +102,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 360 个词/字，references 约 16934 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -156,7 +156,7 @@ }, "contract_boundary": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -184,7 +184,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -206,7 +206,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -216,10 +216,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -271,7 +271,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 360 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -412,7 +412,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 11, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -426,10 +426,7 @@ "status": "pending", "summary_zh": "缺少真实 provider 模型运行和 token metadata。", "summary_en": "Missing a real provider model run and token metadata.", - "blocked_checks": [ - "Provider model run", - "Token usage observed" - ] + "blocked_checks": [] }, { "key": "human-adjudication", @@ -472,7 +469,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -636,17 +634,17 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "logic_steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "usage_steps": [ "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", - "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." ], "package_map": [ { @@ -713,7 +711,7 @@ "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." ], "trigger_contract": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -913,14 +911,14 @@ "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -929,9 +927,19 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "failures": [] }, @@ -1005,43 +1013,43 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d", - "source_contract_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, "command_executed_count": 10, "timing_observed_count": 10, - "model_executed_count": 0, - "token_observed_count": 0, + "model_executed_count": 10, + "token_observed_count": 10, "human_review_complete": false, - "provider_evidence_complete": false, + "provider_evidence_complete": true, "world_class_ready": false, - "world_class_open_gap_count": 4, + "world_class_open_gap_count": 3, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5, + "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 64, - "source_tree_dirty": true, - "source_changed_file_count": 6, + "changed_file_count": 45, + "source_tree_dirty": false, + "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 58 + "generated_changed_file_count": 45 }, - "commit": "ff0473ac02f14a3a431cb01f157453335eb14901", + "commit": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", - "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", "Pending blind-review decisions are visible but do not count as human adjudication.", "World-class readiness remains false until external and human evidence gaps close." ] @@ -1142,7 +1150,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 245, - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" }, "skill_atlas": { "skill_count": 12, @@ -1166,7 +1174,7 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", @@ -1180,7 +1188,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { @@ -1240,7 +1248,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 692, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1300,7 +1308,7 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ @@ -1312,7 +1320,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "to": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" } ] }, @@ -1329,14 +1337,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1345,7 +1353,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1486,15 +1494,15 @@ { "key": "provider-holdout", "label": "Provider Holdout", - "status": "external_required", + "status": "pass", "category": "external", "owner": "operator with provider credentials", - "current": "model-executed 0; token-observed 0", + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1620,7 +1628,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1656,27 +1664,27 @@ "source_audit": { "json": "reports/skill_os2_audit.json", "markdown": "reports/skill_os2_audit.md", - "open_gap_count": 4 + "open_gap_count": 3 } }, "world_class_evidence_ledger": { "ok": true, "summary": { "ledger_entry_count": 4, - "source_accepted_count": 0, + "source_accepted_count": 1, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 4, - "invalid_submission_count": 0, + "missing_submission_count": 3, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 11, + "source_blocked_count": 8, "submitted_but_pending_count": 0, - "source_accepted_without_valid_submission_count": 0, + "source_accepted_without_valid_submission_count": 1, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -1688,14 +1696,14 @@ "category": "external", "owner": "operator with provider credentials", "status": "pending", - "source_status": "external_required", - "source_accepted": false, - "current": "model-executed 0; token-observed 0", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1725,10 +1733,10 @@ "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." ], "observed_state": { - "model_executed_count": 0, + "model_executed_count": 10, "timing_observed_count": 10, - "token_observed_count": 0, - "accepted": false + "token_observed_count": 10, + "accepted": true }, "source_checklist": [ { @@ -1736,9 +1744,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -1748,7 +1756,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -1756,24 +1764,36 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 3, + "source_blocked_count": 0, "submission_state": { - "status": "missing", + "status": "invalid-contract", "path": "evidence/world_class/submissions/provider-holdout.json", - "artifact_ref_count": 0, - "attested_real_evidence": false, - "privacy_contract_satisfied": false, + "submitted_by": "Codex operator using DeepSeek official API", + "submitted_at": "2026-06-17", + "artifact_ref_count": 1, + "artifact_existing_count": 1, + "artifact_sha256_verified_count": 1, + "attested_real_evidence": true, + "reviewer_or_operator_identity_present": true, + "privacy_contract_satisfied": true, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", + "errors": [ + "attestation.ledger_reviewer_approved must be true for a real submission", + "attestation.ledger_reviewer is required", + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + "attestation.ledger_reviewer must be different from submitted_by", + "attestation.ledger_reviewed_at must be at or after submitted_at" + ], "ledger_counts_as_completion": false }, "anti_overclaim": { @@ -2100,7 +2120,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2137,7 +2157,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2158,8 +2178,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2175,8 +2195,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -2212,10 +2232,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -2480,10 +2500,10 @@ "baseline": { "summary": { "target_count": 3, - "baseline_total_errors": 8, - "current_total_errors": 5, - "winner_total_errors": 5, - "winner_vs_baseline_gain": 3, + "baseline_total_errors": 5, + "current_total_errors": 4, + "winner_total_errors": 4, + "winner_vs_baseline_gain": 1, "winner_vs_current_gain": 0 }, "comparisons": [ @@ -2499,7 +2519,7 @@ "adversarial_errors": 0 }, "current": { - "tokens": 65, + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "blind_errors": 0, @@ -2507,7 +2527,7 @@ "adversarial_errors": 0 }, "winner": { - "tokens": 65, + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "blind_errors": 0, @@ -2558,15 +2578,15 @@ "winner_label": "Current", "baseline": { "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2, - "blind_errors": 1, + "dev_errors": 0, + "holdout_errors": 1, + "blind_errors": 0, "judge_blind_errors": 0, "adversarial_errors": 0 }, "current": { "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "blind_errors": 0, "judge_blind_errors": 0, @@ -2574,28 +2594,28 @@ }, "winner": { "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "blind_errors": 0, "judge_blind_errors": 0, "adversarial_errors": 0 }, "delta": { - "current_vs_baseline": 2, - "winner_vs_baseline": 2, + "current_vs_baseline": 0, + "winner_vs_baseline": 0, "winner_vs_current": 0 } } ] }, "compare": { - "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "current_candidate": { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -2611,7 +2631,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -4738,9 +4758,9 @@ "winner": { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -4756,7 +4776,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -4911,18 +4931,16 @@ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", - "score": 0.415, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4932,9 +4950,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -4954,33 +4972,32 @@ { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", - "score": 0.394, + "score": 0.407, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.327, - "support_score": 0, - "lexical_support": 0.417, + "semantic_coverage": 0.333, + "support_score": 0.16, + "lexical_support": 0.25, "negative_penalty": 0, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "eval_optimize": [ @@ -5005,32 +5022,31 @@ { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", - "score": 0.843, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5061,18 +5077,16 @@ { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", - "score": 0.422, + "score": 0.7, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5084,7 +5098,7 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, "lexical_support": 0.4, "negative_penalty": 0, @@ -5110,31 +5124,30 @@ { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", - "score": 0.557, + "score": 0.742, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.16, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, @@ -5161,18 +5174,16 @@ { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", - "score": 0.474, + "score": 0.763, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5183,9 +5194,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0, - "lexical_support": 0.571, + "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5209,32 +5220,31 @@ { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", - "score": 0.868, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0, + "semantic_coverage": 1.0, + "support_score": 0.34, "lexical_support": 0.267, "negative_penalty": 0, "coverage_boost": 0.06, @@ -5267,35 +5277,33 @@ { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", - "score": 0.889, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0.06, - "lexical_support": 0.346, + "semantic_coverage": 1.0, + "support_score": 0.4, + "lexical_support": 0.231, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5342,11 +5350,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5359,9 +5365,9 @@ "exclusive_negative_concepts": [ "brainstorm_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { @@ -5388,11 +5394,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -5429,11 +5433,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5450,9 +5452,9 @@ "brainstorm_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.2, + "lexical_support": 0.133, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { @@ -5486,11 +5488,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -5503,7 +5503,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.167, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { @@ -5527,11 +5527,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5544,9 +5542,9 @@ "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.111, + "lexical_support": 0.222, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { @@ -5573,11 +5571,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5592,9 +5588,9 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.174, + "lexical_support": 0.087, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { @@ -5619,37 +5615,36 @@ { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", - "score": 0.256, + "score": 0.306, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.333, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.222, "negative_penalty": 0.3, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -5684,11 +5679,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5704,7 +5697,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.68, @@ -5739,11 +5732,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5756,9 +5747,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.143, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { @@ -5785,18 +5776,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -5804,11 +5794,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.3, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -5839,18 +5829,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "iterate_existing_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -5859,11 +5848,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.327, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.16, "lexical_support": 0.267, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "iterate_existing_skill": [ @@ -5894,18 +5883,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -5913,11 +5901,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.286, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -5949,18 +5937,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "future_outline", @@ -5970,11 +5957,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.238, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.19, "negative_penalty": 0.82, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -6013,18 +6000,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -6033,11 +6019,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.34, "lexical_support": 0.263, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -6178,18 +6164,16 @@ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", - "score": 0.468, + "score": 0.767, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6202,9 +6186,9 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0.06, - "lexical_support": 0.316, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6231,33 +6215,32 @@ { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", - "score": 0.727, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.691, - "support_score": 0, - "lexical_support": 0.389, + "semantic_coverage": 1.0, + "support_score": 0.16, + "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6288,18 +6271,16 @@ { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", - "score": 0.568, + "score": 0.936, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6310,9 +6291,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, + "semantic_coverage": 0.933, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.214, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6335,34 +6316,32 @@ { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", - "score": 0.858, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0.06, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.4, + "lexical_support": 0.312, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6393,32 +6372,31 @@ { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", - "score": 0.744, + "score": 0.839, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.709, - "support_score": 0, - "lexical_support": 0.4, + "semantic_coverage": 0.733, + "support_score": 0.34, + "lexical_support": 0.24, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6455,11 +6433,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6475,9 +6451,9 @@ "no_build_directive", "summarize_only" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { @@ -6511,11 +6487,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6531,7 +6505,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.68, @@ -6567,11 +6541,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6585,7 +6557,7 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, "lexical_support": 0.222, "negative_penalty": 0.6, @@ -6619,19 +6591,18 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "no_build_directive", "translate_only" @@ -6640,11 +6611,11 @@ "no_build_directive", "translate_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.25, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.188, "negative_penalty": 0.77, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -6682,17 +6653,15 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "matched_desired_concepts": [ + "matched_desired_concepts": [], + "extra_positive_concepts": [ "build_skill" ], - "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" @@ -6701,9 +6670,9 @@ "brainstorm_only", "future_outline" ], - "semantic_coverage": 0.309, - "support_score": 0, - "lexical_support": 0.308, + "semantic_coverage": 0.0, + "support_score": 0.34, + "lexical_support": 0.077, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { @@ -6733,11 +6702,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -6752,7 +6719,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.214, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -6780,18 +6747,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -6799,11 +6765,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.615, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.308, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -6829,26 +6795,25 @@ { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", - "score": 0.0, + "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -6856,11 +6821,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.673, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.34, "lexical_support": 0.438, "negative_penalty": 0.6, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -7033,18 +6998,16 @@ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", - "score": 0.415, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7054,9 +7017,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -7076,33 +7039,32 @@ { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", - "score": 0.394, + "score": 0.407, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.327, - "support_score": 0, - "lexical_support": 0.417, + "semantic_coverage": 0.333, + "support_score": 0.16, + "lexical_support": 0.25, "negative_penalty": 0, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "eval_optimize": [ @@ -7127,32 +7089,31 @@ { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", - "score": 0.843, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -7183,18 +7144,16 @@ { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", - "score": 0.422, + "score": 0.7, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7206,7 +7165,7 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, "lexical_support": 0.4, "negative_penalty": 0, @@ -7232,31 +7191,30 @@ { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", - "score": 0.557, + "score": 0.742, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.16, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, @@ -7283,18 +7241,16 @@ { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", - "score": 0.474, + "score": 0.763, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7305,9 +7261,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0, - "lexical_support": 0.571, + "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -7331,32 +7287,31 @@ { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", - "score": 0.868, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0, + "semantic_coverage": 1.0, + "support_score": 0.34, "lexical_support": 0.267, "negative_penalty": 0, "coverage_boost": 0.06, @@ -7389,35 +7344,33 @@ { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", - "score": 0.889, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0.06, - "lexical_support": 0.346, + "semantic_coverage": 1.0, + "support_score": 0.4, + "lexical_support": 0.231, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -7464,11 +7417,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7481,9 +7432,9 @@ "exclusive_negative_concepts": [ "brainstorm_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { @@ -7510,11 +7461,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -7551,11 +7500,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7572,9 +7519,9 @@ "brainstorm_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.2, + "lexical_support": 0.133, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { @@ -7608,11 +7555,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -7625,7 +7570,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.167, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { @@ -7649,11 +7594,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7666,9 +7609,9 @@ "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.111, + "lexical_support": 0.222, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { @@ -7695,11 +7638,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7714,9 +7655,9 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.174, + "lexical_support": 0.087, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { @@ -7741,37 +7682,36 @@ { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", - "score": 0.256, + "score": 0.306, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.333, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.222, "negative_penalty": 0.3, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -7806,11 +7746,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7826,7 +7764,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.68, @@ -7861,11 +7799,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7878,9 +7814,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.143, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { @@ -7907,18 +7843,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -7926,11 +7861,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.3, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -7961,18 +7896,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "iterate_existing_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -7981,11 +7915,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.327, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.16, "lexical_support": 0.267, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "iterate_existing_skill": [ @@ -8016,18 +7950,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -8035,11 +7968,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.286, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -8071,18 +8004,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "future_outline", @@ -8092,11 +8024,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.238, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.19, "negative_penalty": 0.82, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -8135,18 +8067,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -8155,11 +8086,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.34, "lexical_support": 0.263, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -8300,18 +8231,16 @@ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", - "score": 0.468, + "score": 0.767, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8324,9 +8253,9 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0.06, - "lexical_support": 0.316, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -8353,33 +8282,32 @@ { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", - "score": 0.727, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.691, - "support_score": 0, - "lexical_support": 0.389, + "semantic_coverage": 1.0, + "support_score": 0.16, + "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -8410,18 +8338,16 @@ { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", - "score": 0.568, + "score": 0.936, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8432,9 +8358,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, + "semantic_coverage": 0.933, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.214, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -8457,34 +8383,32 @@ { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", - "score": 0.858, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0.06, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.4, + "lexical_support": 0.312, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -8515,32 +8439,31 @@ { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", - "score": 0.744, + "score": 0.839, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.709, - "support_score": 0, - "lexical_support": 0.4, + "semantic_coverage": 0.733, + "support_score": 0.34, + "lexical_support": 0.24, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -8577,11 +8500,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8597,9 +8518,9 @@ "no_build_directive", "summarize_only" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { @@ -8633,11 +8554,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8653,7 +8572,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.68, @@ -8689,11 +8608,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8707,7 +8624,7 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, "lexical_support": 0.222, "negative_penalty": 0.6, @@ -8741,19 +8658,18 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "no_build_directive", "translate_only" @@ -8762,11 +8678,11 @@ "no_build_directive", "translate_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.25, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.188, "negative_penalty": 0.77, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -8804,17 +8720,15 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "matched_desired_concepts": [ + "matched_desired_concepts": [], + "extra_positive_concepts": [ "build_skill" ], - "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" @@ -8823,9 +8737,9 @@ "brainstorm_only", "future_outline" ], - "semantic_coverage": 0.309, - "support_score": 0, - "lexical_support": 0.308, + "semantic_coverage": 0.0, + "support_score": 0.34, + "lexical_support": 0.077, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { @@ -8855,11 +8769,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -8874,7 +8786,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.214, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -8902,18 +8814,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -8921,11 +8832,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.615, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.308, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -8951,26 +8862,25 @@ { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", - "score": 0.0, + "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -8978,11 +8888,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.673, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.34, "lexical_support": 0.438, "negative_penalty": 0.6, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -9085,18 +8995,16 @@ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", - "score": 0.392, + "score": 0.665, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9106,9 +9014,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.214, + "lexical_support": 0.143, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -9123,24 +9031,21 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", - "score": 0.405, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9150,9 +9055,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.375, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -9168,26 +9073,23 @@ }, "negative": {} } - }, - "boundary_case": true + } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", - "score": 0.028, + "score": 0.16, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9200,9 +9102,9 @@ "document_only" ], "exclusive_negative_concepts": [], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.318, + "lexical_support": 0.227, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { @@ -9232,11 +9134,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -9274,11 +9174,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -9293,7 +9191,7 @@ ], "semantic_coverage": 0.0, "support_score": 0.06, - "lexical_support": 0.308, + "lexical_support": 0.231, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { @@ -9313,18 +9211,16 @@ { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", - "score": 0.026, + "score": 0.305, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9338,7 +9234,7 @@ "exclusive_negative_concepts": [ "future_outline" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.22, @@ -9358,7 +9254,8 @@ ] } } - } + }, + "boundary_case": true } ] } @@ -9437,18 +9334,16 @@ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", - "score": 0.392, + "score": 0.665, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9458,9 +9353,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.214, + "lexical_support": 0.143, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -9475,24 +9370,21 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", - "score": 0.405, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9502,9 +9394,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.375, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -9520,26 +9412,23 @@ }, "negative": {} } - }, - "boundary_case": true + } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", - "score": 0.028, + "score": 0.16, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9552,9 +9441,9 @@ "document_only" ], "exclusive_negative_concepts": [], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.318, + "lexical_support": 0.227, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { @@ -9584,11 +9473,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -9626,11 +9513,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -9645,7 +9530,7 @@ ], "semantic_coverage": 0.0, "support_score": 0.06, - "lexical_support": 0.308, + "lexical_support": 0.231, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { @@ -9665,18 +9550,16 @@ { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", - "score": 0.026, + "score": 0.305, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9690,7 +9573,7 @@ "exclusive_negative_concepts": [ "future_outline" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.22, @@ -9710,7 +9593,8 @@ ] } } - } + }, + "boundary_case": true } ] } @@ -11176,32 +11060,31 @@ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", - "score": 0.834, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.263, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -11228,32 +11111,31 @@ { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", - "score": 0.842, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.368, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -11288,11 +11170,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -11309,9 +11189,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, - "lexical_support": 0.429, + "lexical_support": 0.357, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { @@ -11347,11 +11227,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -11369,9 +11247,9 @@ "summarize_only", "translate_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.188, + "lexical_support": 0.125, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { @@ -11410,11 +11288,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -11429,7 +11305,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.2, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -11458,11 +11334,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -11479,9 +11353,9 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.182, + "lexical_support": 0.273, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { @@ -11581,32 +11455,31 @@ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", - "score": 0.834, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.263, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -11633,32 +11506,31 @@ { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", - "score": 0.842, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.368, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -11693,11 +11565,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -11714,9 +11584,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, - "lexical_support": 0.429, + "lexical_support": 0.357, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { @@ -11752,11 +11622,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -11774,9 +11642,9 @@ "summarize_only", "translate_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.188, + "lexical_support": 0.125, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { @@ -11815,11 +11683,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -11834,7 +11700,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.2, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -11863,11 +11729,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -11884,9 +11748,9 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.182, + "lexical_support": 0.273, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { @@ -12315,9 +12179,9 @@ { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -12333,7 +12197,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -12667,30 +12531,30 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" @@ -12768,33 +12632,33 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "baseline_calibration": { "threshold": 0.33, @@ -12943,14 +12807,14 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -12958,14 +12822,14 @@ }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -13020,8 +12884,8 @@ }, "summary": { "winner_label": "Current", - "winner_tokens": 65, - "current_tokens": 65, + "winner_tokens": 53, + "current_tokens": 53, "winner_dev_total_errors": 0, "current_dev_total_errors": 0, "winner_holdout_total_errors": 0, @@ -13034,7 +12898,7 @@ "current_adversarial_holdout_total_errors": 0, "winner_judge_blind_agreement_rate": 1.0, "winner_adversarial_risk_band": "healthy", - "winner_adversarial_score_gap": 0.834, + "winner_adversarial_score_gap": 1.0, "candidate_count": 6, "baseline_tokens": 8, "baseline_dev_total_errors": 1, @@ -13055,9 +12919,9 @@ { "name": "yao-meta-skill", "winner_label": "Current", - "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "winner_tokens": 65, - "current_tokens": 65, + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "winner_tokens": 53, + "current_tokens": 53, "winner_holdout_fp": 0, "winner_holdout_fn": 0, "current_holdout_fp": 0, @@ -13088,44 +12952,44 @@ "calibration": { "holdout": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "blind_holdout": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "adversarial_holdout": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -13216,7 +13080,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.611, + "average_margin": 0.701, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -13239,8 +13103,8 @@ "label": "Current", "ranking_state": "selected_by_dev", "promotion_state": "kept_current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "tokens": 65, + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "reason_tags": [] @@ -13270,7 +13134,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -13284,7 +13149,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -13584,13 +13450,13 @@ "current_holdout_fp": 0, "current_holdout_fn": 1, "baseline_holdout_fp": 0, - "baseline_holdout_fn": 2, + "baseline_holdout_fn": 1, "winner_blind_holdout_fp": 0, "winner_blind_holdout_fn": 0, "current_blind_holdout_fp": 0, "current_blind_holdout_fn": 0, "baseline_blind_holdout_fp": 0, - "baseline_blind_holdout_fn": 1, + "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_fp": 0, "winner_judge_blind_holdout_fn": 0, @@ -13609,7 +13475,7 @@ "calibration": { "holdout": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -13624,49 +13490,49 @@ }, "blind_holdout": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "adversarial_holdout": { "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" } }, "judge_blind": { "winner": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "current": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "baseline": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -13729,7 +13595,8 @@ "causes": [ "no_candidate_outperformed_current", "current_holdout_gap_present", - "current_holdout_risk" + "current_holdout_risk", + "adversarial_watch_risk" ], "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", "gate_status": { @@ -13745,7 +13612,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.659, + "average_margin": 0.586, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -13759,8 +13626,8 @@ "promotion_state": "reference", "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2 + "dev_errors": 0, + "holdout_errors": 1 }, { "name": "governed-incident-command", @@ -13770,25 +13637,10 @@ "promotion_state": "kept_current", "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [] }, - { - "name": "governed-incident-command", - "role": "candidate", - "label": "Guardrail", - "ranking_state": "not_selected", - "promotion_state": "blocked", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", - "tokens": 51, - "dev_errors": 1, - "holdout_errors": 2, - "reason_tags": [ - "visible_holdout_regression", - "longer_without_gain" - ] - }, { "name": "governed-incident-command", "role": "candidate", @@ -13797,7 +13649,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 54, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -13811,7 +13663,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", "tokens": 78, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -13825,9 +13677,24 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 78, + "dev_errors": 0, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, "dev_errors": 1, "holdout_errors": 1, "reason_tags": [ + "weaker_dev_fit", "longer_without_gain" ] } @@ -14026,7 +13893,7 @@ }, "reference_synthesis": { "skill_name": "yao-meta-skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "intent_confidence": { "score": 100, "band": "high", @@ -14434,16 +14301,40 @@ "skill_name": "yao-meta-skill", "design_system": "metric editorial", "primary_artifact": { - "key": "code_or_cli", - "label": "Code, CLI, or implementation guide", - "direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", + "key": "dashboard", + "label": "Dashboard or metrics page", + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", "matched_keywords": [ - "code", - "script", - "command" + "metric", + "score", + "table", + "scorecard" ] }, "artifact_families": [ + { + "key": "dashboard", + "label": "Dashboard or metrics page", + "score": 4, + "matched_keywords": [ + "metric", + "score", + "table", + "scorecard" + ], + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." + }, + { + "key": "review_viewer", + "label": "Review viewer", + "score": 3, + "matched_keywords": [ + "review", + "viewer", + "audit" + ], + "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." + }, { "key": "code_or_cli", "label": "Code, CLI, or implementation guide", @@ -14465,25 +14356,6 @@ ], "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." }, - { - "key": "review_viewer", - "label": "Review viewer", - "score": 2, - "matched_keywords": [ - "review", - "viewer" - ], - "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." - }, - { - "key": "dashboard", - "label": "Dashboard or metrics page", - "score": 1, - "matched_keywords": [ - "table" - ], - "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." - }, { "key": "visual_capture", "label": "Screenshot or visual evidence", @@ -14495,14 +14367,14 @@ } ], "layout_patterns": [ - "prerequisites", - "commands", - "expected output", - "failure handling", - "rollback or cleanup", - "thesis", - "evidence blocks", - "decision table" + "metric board", + "ranked signals", + "comparison rows", + "interpretation", + "action queue", + "summary", + "variant comparison", + "evidence" ], "design_tokens": { "type": [ @@ -14527,14 +14399,14 @@ ] }, "quality_gates": [ - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding.", - "Keep the first screen useful without requiring the reader to parse every detail.", - "Use tables only for comparisons; move explanations below the table.", - "Keep source notes readable without flooding the body with markers.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities.", "Make differences visible instead of hiding them in prose.", - "Separate author-facing recommendations from reviewer-only evidence." + "Separate author-facing recommendations from reviewer-only evidence.", + "Surface conflicts clearly and keep routine benchmark synthesis quiet.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations." ], "anti_patterns": [ "Do not copy Kami's fixed parchment background as a default.", @@ -14569,18 +14441,6 @@ "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." }, - { - "key": "dialogue_interaction", - "label": "Dialogue interaction", - "score": 2, - "matched_keywords": [ - "dialogue", - "conversation" - ], - "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", - "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", - "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." - }, { "key": "creative_generation", "label": "Creative generation", @@ -14602,6 +14462,17 @@ "role_guidance": "Use an analyst role that separates evidence, inference, uncertainty, and recommendation.", "task_guidance": "State assumptions, compare alternatives, and make the decision path inspectable.", "format_guidance": "Return findings, evidence, tradeoffs, recommendation, and residual risks." + }, + { + "key": "dialogue_interaction", + "label": "Dialogue interaction", + "score": 1, + "matched_keywords": [ + "dialogue" + ], + "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", + "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", + "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." } ], "complexity": { @@ -14627,6 +14498,7 @@ "label": "Completeness", "score": 100, "matched_signals": [ + "input", "output", "constraint", "standard" @@ -14711,7 +14583,7 @@ { "label": "Artifact design profile", "status": "ready", - "detail": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit." + "detail": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." }, { "label": "Prompt quality profile", diff --git a/reports/review_annotations.json b/reports/review_annotations.json index 66f2b1d9..76fadc1f 100644 --- a/reports/review_annotations.json +++ b/reports/review_annotations.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, diff --git a/reports/review_waivers.json b/reports/review_waivers.json index f39abd11..aaa4a9d5 100644 --- a/reports/review_waivers.json +++ b/reports/review_waivers.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "summary": { "waiver_count": 0, "active_count": 0, @@ -92,7 +92,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { diff --git a/reports/review_waivers.md b/reports/review_waivers.md index e70204a5..c303e8bf 100644 --- a/reports/review_waivers.md +++ b/reports/review_waivers.md @@ -38,7 +38,7 @@ - waiver allowed: `true` - risk: review pending 5; model-executed 10; output failures 0 - evidence: `reports/output_review_adjudication.md` -- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md` +- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md` - world-class boundary: Does not count as provider, human, or public world-class completion evidence. #### Required Review diff --git a/reports/skill-interpretation.html b/reports/skill-interpretation.html index b933f181..8366ba87 100644 --- a/reports/skill-interpretation.html +++ b/reports/skill-interpretation.html @@ -761,7 +761,7 @@

YAO Skill 生成审计报告YAO Skill Generation Audit

yao-meta-skill

技能审计报告Yao Meta Skill Audit Report

-

这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

技能名称：yao-meta-skillSkill name: yao-meta-skill成熟度：治理Maturity: governed格式：Agent SkillsFormat: agent-skills更新时间：2026-03-31Updated: 2026-03-31
openaiopenaiclaudeclaudegenericgenericvscodevscode
@@ -770,7 +770,7 @@
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • Skill 作者、复用团队和后续 reviewer。Skill authors, reuse teams, and later reviewers.
  • 创建完成后建议先打开 reports/skill-interpretation.html，再继续扩展包体。After creation, open reports/skill-interpretation.html before expanding the package further.
-
完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
上下文成本Context cost42入口约 360 个词/字，references 约 16934 个词/字。Entrypoint is about 360 words/characters; references are about 16934.
+
完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
上下文成本Context cost42入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
@@ -782,7 +782,7 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • 交付结果：SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html, reports/skill-interpretation.jsonDeliverables: SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html, reports/skill-interpretation.json
+
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • 交付结果：SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html, reports/skill-interpretation.jsonDeliverables: SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html, reports/skill-interpretation.json
交付流程Delivery Flow输入材料Input materialSkill 包体Skill package可复用能力Reusable capability
交付流程把用户输入、生成的包体和可复用能力放在一条线上。The delivery flow places user input, generated package, and reusable capability on one path.
@@ -807,12 +807,12 @@

成熟度条Maturity Bar

这里把每个指标压缩为状态、分数和第一条证据，先给出整体判断，再进入下方明细。This compresses each metric into status, score, and the first evidence item before the detailed cards below.

-
  1. 稳定Stable完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  2. 稳定Stable触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  3. 稳定Stable证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  4. 稳定Stable可维护性Maintainability100SKILL.md 约 360 个词/字。SKILL.md is about 360 words/characters.
  5. 稳定Stable可迁移性Portability100agents/interface.yaml 已存在。agents/interface.yaml exists.
  6. 关注Watch上下文成本Context cost42入口约 360 个词/字，references 约 16934 个词/字。Entrypoint is about 360 words/characters; references are about 16934.
+
  1. 稳定Stable完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  2. 稳定Stable触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  3. 稳定Stable证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  4. 稳定Stable可维护性Maintainability100SKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.
  5. 稳定Stable可迁移性Portability100agents/interface.yaml 已存在。agents/interface.yaml exists.
  6. 关注Watch上下文成本Context cost42入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
指标明细Metric Details
-
完整度Completeness100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  • README.md 已存在，便于人工阅读。README.md exists for human-readable usage.
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml exists for cross-platform adaptation.
触发清晰Trigger clarity100
  • frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  • description 有足够长度说明任务边界。The description is long enough to explain the task boundary.
  • description 已包含使用场景或排除边界信号。The description includes usage-scenario or exclusion-boundary signals.
证据充分Evidence depth100
  • 已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  • skill-ir.json 已存在。skill-ir.json exists.
  • compiled_targets.json 已存在。compiled_targets.json exists.
可维护性Maintainability100
  • SKILL.md 约 360 个词/字。SKILL.md is about 360 words/characters.
  • 入口文件保持克制，可维护性较好。The entrypoint stays restrained, which supports maintainability.
  • references/ 已承载扩展指导。references/ carries extended guidance.
可迁移性Portability100
  • agents/interface.yaml 已存在。agents/interface.yaml exists.
  • manifest.json 已存在。manifest.json exists.
  • 目标平台或 adapter target 已声明。Target platforms or adapter targets are declared.
上下文成本Context cost42
  • 入口约 360 个词/字，references 约 16934 个词/字。Entrypoint is about 360 words/characters; references are about 16934.
  • 分数越高代表上下文成本越低。A higher score means lower context cost.
  • 上下文成本偏高，建议压缩入口或拆分 references。Context cost is high; compress the entrypoint or split references further.
+
完整度Completeness100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  • README.md 已存在，便于人工阅读。README.md exists for human-readable usage.
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml exists for cross-platform adaptation.
触发清晰Trigger clarity100
  • frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  • description 有足够长度说明任务边界。The description is long enough to explain the task boundary.
  • description 已包含使用场景或排除边界信号。The description includes usage-scenario or exclusion-boundary signals.
证据充分Evidence depth100
  • 已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  • skill-ir.json 已存在。skill-ir.json exists.
  • compiled_targets.json 已存在。compiled_targets.json exists.
可维护性Maintainability100
  • SKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.
  • 入口文件保持克制，可维护性较好。The entrypoint stays restrained, which supports maintainability.
  • references/ 已承载扩展指导。references/ carries extended guidance.
可迁移性Portability100
  • agents/interface.yaml 已存在。agents/interface.yaml exists.
  • manifest.json 已存在。manifest.json exists.
  • 目标平台或 adapter target 已声明。Target platforms or adapter targets are declared.
上下文成本Context cost42
  • 入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
  • 分数越高代表上下文成本越低。A higher score means lower context cost.
  • 上下文成本偏高，建议压缩入口或拆分 references。Context cost is high; compress the entrypoint or split references further.
@@ -845,7 +845,7 @@

执行流程Execution Flow

-
  1. 判断请求是否应该沉淀为 Skill，并选择最轻量可靠的模式。Decide whether the request should become a skill and choose the lightest fit.
  2. 捕捉任务、输出、排除项、约束和质量标准。Capture job, output, exclusions, constraints, and standards.
  3. 运行参考扫描：先看外部 benchmark，再看用户材料，最后校验本地适配；只暴露不确定性或冲突。Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
  4. 尽早写出 `description`，先测试路由质量，再扩展包体。Write the `description` early and test route quality before expanding the package.
  5. 只在确有价值时添加 output-risk、artifact-design、prompt-quality 和 system-model 报告。Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.
+
  1. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.
  2. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Capture job, output, exclusions, constraints, standards, and the lightest fit.
  3. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.
  4. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Write `description` early, test route quality, then add only earned folders and gates.
  5. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful.

调用方式How To Use

@@ -865,7 +865,7 @@

触发描述Trigger

-

该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

输入材料Inputs

  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。an existing skill directory that needs refactor, evaluation, packaging, or governance hardening
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。benchmark references, local constraints, desired maturity tier, and review standards
@@ -888,12 +888,12 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.保留并复用Keep
缺口Gap上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。Context cost needs improvement: Entrypoint is about 360 words/characters; references are about 16934.纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.保留并复用Keep
缺口Gap上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。Context cost needs improvement: Entrypoint is about 339 words/characters; references are about 16934.纳入下一轮修复Fix next

执行证据Execution Evidence

-
  • 变体运行：10Variant runs: 10
  • 模型执行：0Model executions: 0
  • 记录样本：0Recorded fixtures: 0
  • Token 估算：10Token estimates: 10
+
  • 变体运行：10Variant runs: 10
  • 模型执行：10Model executions: 10
  • 记录样本：0Recorded fixtures: 0
  • Token 估算：0Token estimates: 0

盲评审定Blind Adjudication

@@ -925,11 +925,11 @@
- +
风险Risk信号Signal应对Response
误触发风险Trigger riskfrontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
输出漂移风险Output drift risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
证据不足风险Evidence gap risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
包体膨胀风险Package bloat riskSKILL.md 约 360 个词/字。SKILL.md is about 360 words/characters.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
跨平台迁移风险Portability riskagents/interface.yaml 已存在。agents/interface.yaml exists.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
误触发风险Trigger riskfrontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
输出漂移风险Output drift risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
证据不足风险Evidence gap risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
包体膨胀风险Package bloat riskSKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
跨平台迁移风险Portability riskagents/interface.yaml 已存在。agents/interface.yaml exists.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
-

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks10 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks11 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks

当前没有阻塞检查。No blocked checks right now.

人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index c91a7f3d..84ef5cb5 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -3,7 +3,7 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "maturity": "governed", "updated_at": "2026-03-31", "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", @@ -81,7 +81,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 360 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -102,7 +102,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 360 个词/字，references 约 16934 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -156,7 +156,7 @@ }, "contract_boundary": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -184,7 +184,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -206,7 +206,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -216,10 +216,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -271,7 +271,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 360 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -412,7 +412,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 11, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -426,10 +426,7 @@ "status": "pending", "summary_zh": "缺少真实 provider 模型运行和 token metadata。", "summary_en": "Missing a real provider model run and token metadata.", - "blocked_checks": [ - "Provider model run", - "Token usage observed" - ] + "blocked_checks": [] }, { "key": "human-adjudication", @@ -472,7 +469,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -640,17 +638,17 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "logic_steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "usage_steps": [ "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", - "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." ], "package_map": [ { @@ -717,7 +715,7 @@ "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." ], "trigger_contract": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -917,14 +915,14 @@ "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -933,9 +931,19 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "failures": [] }, @@ -1009,43 +1017,43 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d", - "source_contract_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, "command_executed_count": 10, "timing_observed_count": 10, - "model_executed_count": 0, - "token_observed_count": 0, + "model_executed_count": 10, + "token_observed_count": 10, "human_review_complete": false, - "provider_evidence_complete": false, + "provider_evidence_complete": true, "world_class_ready": false, - "world_class_open_gap_count": 4, + "world_class_open_gap_count": 3, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5, + "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 64, - "source_tree_dirty": true, - "source_changed_file_count": 6, + "changed_file_count": 45, + "source_tree_dirty": false, + "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 58 + "generated_changed_file_count": 45 }, - "commit": "ff0473ac02f14a3a431cb01f157453335eb14901", + "commit": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", - "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", "Pending blind-review decisions are visible but do not count as human adjudication.", "World-class readiness remains false until external and human evidence gaps close." ] @@ -1146,7 +1154,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 245, - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" }, "skill_atlas": { "skill_count": 12, @@ -1170,7 +1178,7 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", @@ -1184,7 +1192,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { @@ -1244,7 +1252,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 692, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1304,7 +1312,7 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ @@ -1316,7 +1324,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "to": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" } ] }, @@ -1333,14 +1341,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1349,7 +1357,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1490,15 +1498,15 @@ { "key": "provider-holdout", "label": "Provider Holdout", - "status": "external_required", + "status": "pass", "category": "external", "owner": "operator with provider credentials", - "current": "model-executed 0; token-observed 0", + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1624,7 +1632,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1660,27 +1668,27 @@ "source_audit": { "json": "reports/skill_os2_audit.json", "markdown": "reports/skill_os2_audit.md", - "open_gap_count": 4 + "open_gap_count": 3 } }, "world_class_evidence_ledger": { "ok": true, "summary": { "ledger_entry_count": 4, - "source_accepted_count": 0, + "source_accepted_count": 1, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 4, - "invalid_submission_count": 0, + "missing_submission_count": 3, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 11, + "source_blocked_count": 8, "submitted_but_pending_count": 0, - "source_accepted_without_valid_submission_count": 0, + "source_accepted_without_valid_submission_count": 1, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -1692,14 +1700,14 @@ "category": "external", "owner": "operator with provider credentials", "status": "pending", - "source_status": "external_required", - "source_accepted": false, - "current": "model-executed 0; token-observed 0", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1729,10 +1737,10 @@ "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." ], "observed_state": { - "model_executed_count": 0, + "model_executed_count": 10, "timing_observed_count": 10, - "token_observed_count": 0, - "accepted": false + "token_observed_count": 10, + "accepted": true }, "source_checklist": [ { @@ -1740,9 +1748,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -1752,7 +1760,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -1760,24 +1768,36 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 3, + "source_blocked_count": 0, "submission_state": { - "status": "missing", + "status": "invalid-contract", "path": "evidence/world_class/submissions/provider-holdout.json", - "artifact_ref_count": 0, - "attested_real_evidence": false, - "privacy_contract_satisfied": false, + "submitted_by": "Codex operator using DeepSeek official API", + "submitted_at": "2026-06-17", + "artifact_ref_count": 1, + "artifact_existing_count": 1, + "artifact_sha256_verified_count": 1, + "attested_real_evidence": true, + "reviewer_or_operator_identity_present": true, + "privacy_contract_satisfied": true, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", + "errors": [ + "attestation.ledger_reviewer_approved must be true for a real submission", + "attestation.ledger_reviewer is required", + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + "attestation.ledger_reviewer must be different from submitted_by", + "attestation.ledger_reviewed_at must be at or after submitted_at" + ], "ledger_counts_as_completion": false }, "anti_overclaim": { @@ -2104,7 +2124,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2141,7 +2161,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2162,8 +2182,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2179,8 +2199,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -2216,10 +2236,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { diff --git a/reports/skill-overview.html b/reports/skill-overview.html index c2a617b3..7448c1c8 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -761,7 +761,7 @@

YAO Skill 生成审计报告YAO Skill Generation Audit

yao-meta-skill

技能审计报告Yao Meta Skill Audit Report

-

这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

技能名称：yao-meta-skillSkill name: yao-meta-skill成熟度：治理Maturity: governed格式：Agent SkillsFormat: agent-skills更新时间：2026-03-31Updated: 2026-03-31
openaiopenaiclaudeclaudegenericgenericvscodevscode
@@ -770,7 +770,7 @@
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • Skill 作者、复用团队和后续 reviewer。Skill authors, reuse teams, and later reviewers.
  • 创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。After creation, open reports/skill-overview.html before expanding the package further.
-
完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
上下文成本Context cost42入口约 360 个词/字，references 约 16934 个词/字。Entrypoint is about 360 words/characters; references are about 16934.
+
完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
上下文成本Context cost42入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
@@ -782,7 +782,7 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • 交付结果：SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.htmlDeliverables: SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html
+
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • 交付结果：SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.htmlDeliverables: SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html
交付流程Delivery Flow输入材料Input materialSkill 包体Skill package可复用能力Reusable capability
交付流程把用户输入、生成的包体和可复用能力放在一条线上。The delivery flow places user input, generated package, and reusable capability on one path.
@@ -807,12 +807,12 @@

成熟度条Maturity Bar

这里把每个指标压缩为状态、分数和第一条证据，先给出整体判断，再进入下方明细。This compresses each metric into status, score, and the first evidence item before the detailed cards below.

-
  1. 稳定Stable完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  2. 稳定Stable触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  3. 稳定Stable证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  4. 稳定Stable可维护性Maintainability100SKILL.md 约 360 个词/字。SKILL.md is about 360 words/characters.
  5. 稳定Stable可迁移性Portability100agents/interface.yaml 已存在。agents/interface.yaml exists.
  6. 关注Watch上下文成本Context cost42入口约 360 个词/字，references 约 16934 个词/字。Entrypoint is about 360 words/characters; references are about 16934.
+
  1. 稳定Stable完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  2. 稳定Stable触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  3. 稳定Stable证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  4. 稳定Stable可维护性Maintainability100SKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.
  5. 稳定Stable可迁移性Portability100agents/interface.yaml 已存在。agents/interface.yaml exists.
  6. 关注Watch上下文成本Context cost42入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
指标明细Metric Details
-
完整度Completeness100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  • README.md 已存在，便于人工阅读。README.md exists for human-readable usage.
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml exists for cross-platform adaptation.
触发清晰Trigger clarity100
  • frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  • description 有足够长度说明任务边界。The description is long enough to explain the task boundary.
  • description 已包含使用场景或排除边界信号。The description includes usage-scenario or exclusion-boundary signals.
证据充分Evidence depth100
  • 已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  • skill-ir.json 已存在。skill-ir.json exists.
  • compiled_targets.json 已存在。compiled_targets.json exists.
可维护性Maintainability100
  • SKILL.md 约 360 个词/字。SKILL.md is about 360 words/characters.
  • 入口文件保持克制，可维护性较好。The entrypoint stays restrained, which supports maintainability.
  • references/ 已承载扩展指导。references/ carries extended guidance.
可迁移性Portability100
  • agents/interface.yaml 已存在。agents/interface.yaml exists.
  • manifest.json 已存在。manifest.json exists.
  • 目标平台或 adapter target 已声明。Target platforms or adapter targets are declared.
上下文成本Context cost42
  • 入口约 360 个词/字，references 约 16934 个词/字。Entrypoint is about 360 words/characters; references are about 16934.
  • 分数越高代表上下文成本越低。A higher score means lower context cost.
  • 上下文成本偏高，建议压缩入口或拆分 references。Context cost is high; compress the entrypoint or split references further.
+
完整度Completeness100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  • README.md 已存在，便于人工阅读。README.md exists for human-readable usage.
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml exists for cross-platform adaptation.
触发清晰Trigger clarity100
  • frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  • description 有足够长度说明任务边界。The description is long enough to explain the task boundary.
  • description 已包含使用场景或排除边界信号。The description includes usage-scenario or exclusion-boundary signals.
证据充分Evidence depth100
  • 已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  • skill-ir.json 已存在。skill-ir.json exists.
  • compiled_targets.json 已存在。compiled_targets.json exists.
可维护性Maintainability100
  • SKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.
  • 入口文件保持克制，可维护性较好。The entrypoint stays restrained, which supports maintainability.
  • references/ 已承载扩展指导。references/ carries extended guidance.
可迁移性Portability100
  • agents/interface.yaml 已存在。agents/interface.yaml exists.
  • manifest.json 已存在。manifest.json exists.
  • 目标平台或 adapter target 已声明。Target platforms or adapter targets are declared.
上下文成本Context cost42
  • 入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
  • 分数越高代表上下文成本越低。A higher score means lower context cost.
  • 上下文成本偏高，建议压缩入口或拆分 references。Context cost is high; compress the entrypoint or split references further.
@@ -845,7 +845,7 @@

执行流程Execution Flow

-
  1. 判断请求是否应该沉淀为 Skill，并选择最轻量可靠的模式。Decide whether the request should become a skill and choose the lightest fit.
  2. 捕捉任务、输出、排除项、约束和质量标准。Capture job, output, exclusions, constraints, and standards.
  3. 运行参考扫描：先看外部 benchmark，再看用户材料，最后校验本地适配；只暴露不确定性或冲突。Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
  4. 尽早写出 `description`，先测试路由质量，再扩展包体。Write the `description` early and test route quality before expanding the package.
  5. 只在确有价值时添加 output-risk、artifact-design、prompt-quality 和 system-model 报告。Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.
+
  1. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.
  2. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Capture job, output, exclusions, constraints, standards, and the lightest fit.
  3. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.
  4. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Write `description` early, test route quality, then add only earned folders and gates.
  5. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful.

调用方式How To Use

@@ -865,7 +865,7 @@

触发描述Trigger

-

该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

输入材料Inputs

  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。an existing skill directory that needs refactor, evaluation, packaging, or governance hardening
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。benchmark references, local constraints, desired maturity tier, and review standards
@@ -888,12 +888,12 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.保留并复用Keep
缺口Gap上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。Context cost needs improvement: Entrypoint is about 360 words/characters; references are about 16934.纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.保留并复用Keep
缺口Gap上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。Context cost needs improvement: Entrypoint is about 339 words/characters; references are about 16934.纳入下一轮修复Fix next

执行证据Execution Evidence

-
  • 变体运行：10Variant runs: 10
  • 模型执行：0Model executions: 0
  • 记录样本：0Recorded fixtures: 0
  • Token 估算：10Token estimates: 10
+
  • 变体运行：10Variant runs: 10
  • 模型执行：10Model executions: 10
  • 记录样本：0Recorded fixtures: 0
  • Token 估算：0Token estimates: 0

盲评审定Blind Adjudication

@@ -925,11 +925,11 @@
- +
风险Risk信号Signal应对Response
误触发风险Trigger riskfrontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
输出漂移风险Output drift risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
证据不足风险Evidence gap risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
包体膨胀风险Package bloat riskSKILL.md 约 360 个词/字。SKILL.md is about 360 words/characters.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
跨平台迁移风险Portability riskagents/interface.yaml 已存在。agents/interface.yaml exists.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
误触发风险Trigger riskfrontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
输出漂移风险Output drift risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
证据不足风险Evidence gap risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
包体膨胀风险Package bloat riskSKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
跨平台迁移风险Portability riskagents/interface.yaml 已存在。agents/interface.yaml exists.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
-

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks10 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks11 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks

当前没有阻塞检查。No blocked checks right now.

人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index c0b07b31..eed6d3a0 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -3,7 +3,7 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "maturity": "governed", "updated_at": "2026-03-31", "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", @@ -80,7 +80,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 360 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -101,7 +101,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 360 个词/字，references 约 16934 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -155,7 +155,7 @@ }, "contract_boundary": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -183,7 +183,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -205,7 +205,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 360 个词/字，references 约 16934 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -215,10 +215,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -270,7 +270,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 360 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -411,7 +411,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 11, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -425,10 +425,7 @@ "status": "pending", "summary_zh": "缺少真实 provider 模型运行和 token metadata。", "summary_en": "Missing a real provider model run and token metadata.", - "blocked_checks": [ - "Provider model run", - "Token usage observed" - ] + "blocked_checks": [] }, { "key": "human-adjudication", @@ -471,7 +468,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -635,17 +633,17 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "logic_steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "usage_steps": [ "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", - "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." ], "package_map": [ { @@ -712,7 +710,7 @@ "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." ], "trigger_contract": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -912,14 +910,14 @@ "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -928,9 +926,19 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "failures": [] }, @@ -1004,43 +1012,43 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9a91175581e3285aebba447f68246d86af01e3f7bc5e541ca572cce12d3c500d", - "source_contract_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, "command_executed_count": 10, "timing_observed_count": 10, - "model_executed_count": 0, - "token_observed_count": 0, + "model_executed_count": 10, + "token_observed_count": 10, "human_review_complete": false, - "provider_evidence_complete": false, + "provider_evidence_complete": true, "world_class_ready": false, - "world_class_open_gap_count": 4, + "world_class_open_gap_count": 3, "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, "public_claim_ready": false, - "public_claim_blocker_count": 5, + "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 64, - "source_tree_dirty": true, - "source_changed_file_count": 6, + "changed_file_count": 45, + "source_tree_dirty": false, + "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 58 + "generated_changed_file_count": 45 }, - "commit": "ff0473ac02f14a3a431cb01f157453335eb14901", + "commit": "cece6f9b04869e46db9aed1bf0206ccb8fb887e4", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", - "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", "Pending blind-review decisions are visible but do not count as human adjudication.", "World-class readiness remains false until external and human evidence gaps close." ] @@ -1141,7 +1149,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 245, - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" }, "skill_atlas": { "skill_count": 12, @@ -1165,7 +1173,7 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", @@ -1179,7 +1187,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291", + "package_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { @@ -1239,7 +1247,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 692, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1299,7 +1307,7 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ @@ -1311,7 +1319,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7958467751730998d7f2a8f38c9e8ef324874459e62f0c0a197232cd8386b291" + "to": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111" } ] }, @@ -1328,14 +1336,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1344,7 +1352,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1485,15 +1493,15 @@ { "key": "provider-holdout", "label": "Provider Holdout", - "status": "external_required", + "status": "pass", "category": "external", "owner": "operator with provider credentials", - "current": "model-executed 0; token-observed 0", + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1619,7 +1627,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1655,27 +1663,27 @@ "source_audit": { "json": "reports/skill_os2_audit.json", "markdown": "reports/skill_os2_audit.md", - "open_gap_count": 4 + "open_gap_count": 3 } }, "world_class_evidence_ledger": { "ok": true, "summary": { "ledger_entry_count": 4, - "source_accepted_count": 0, + "source_accepted_count": 1, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 4, - "invalid_submission_count": 0, + "missing_submission_count": 3, + "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 11, + "source_blocked_count": 8, "submitted_but_pending_count": 0, - "source_accepted_without_valid_submission_count": 0, + "source_accepted_without_valid_submission_count": 1, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -1687,14 +1695,14 @@ "category": "external", "owner": "operator with provider credentials", "status": "pending", - "source_status": "external_required", - "source_accepted": false, - "current": "model-executed 0; token-observed 0", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" @@ -1724,10 +1732,10 @@ "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." ], "observed_state": { - "model_executed_count": 0, + "model_executed_count": 10, "timing_observed_count": 10, - "token_observed_count": 0, - "accepted": false + "token_observed_count": 10, + "accepted": true }, "source_checklist": [ { @@ -1735,9 +1743,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -1747,7 +1755,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -1755,24 +1763,36 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 3, + "source_blocked_count": 0, "submission_state": { - "status": "missing", + "status": "invalid-contract", "path": "evidence/world_class/submissions/provider-holdout.json", - "artifact_ref_count": 0, - "attested_real_evidence": false, - "privacy_contract_satisfied": false, + "submitted_by": "Codex operator using DeepSeek official API", + "submitted_at": "2026-06-17", + "artifact_ref_count": 1, + "artifact_existing_count": 1, + "artifact_sha256_verified_count": 1, + "attested_real_evidence": true, + "reviewer_or_operator_identity_present": true, + "privacy_contract_satisfied": true, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", + "errors": [ + "attestation.ledger_reviewer_approved must be true for a real submission", + "attestation.ledger_reviewer is required", + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + "attestation.ledger_reviewer must be different from submitted_by", + "attestation.ledger_reviewed_at must be at or after submitted_at" + ], "ledger_counts_as_completion": false }, "anti_overclaim": { @@ -2099,7 +2119,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2136,7 +2156,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2157,8 +2177,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2174,8 +2194,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -2211,10 +2231,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 9eb19c92..96ff4372 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "item_count": 15, @@ -351,7 +351,7 @@ "key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "external_required", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "target": "A real Browser/Chrome/provider client sends production metadata events", "evidence": [ { @@ -415,7 +415,7 @@ "key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "external_required", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "target": "A real Browser/Chrome/provider client sends production metadata events", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 0af55162..60878f68 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -1,6 +1,6 @@ # Skill OS 2.0 Audit -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary @@ -30,7 +30,7 @@ Generated at: `2026-06-13` | Registry Distribution | pass | zip entries 689; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | -| Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | +| Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | ## Open Highest-Leverage Gaps diff --git a/reports/skillops/daily/2026-06-17.json b/reports/skillops/daily/2026-06-17.json index e12bbe23..195a63d7 100644 --- a/reports/skillops/daily/2026-06-17.json +++ b/reports/skillops/daily/2026-06-17.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T04:04:43Z", + "generated_at": "2026-06-17T07:53:10Z", "skill_dir": ".", "decision": "proposal-review", "source_supplied": false, @@ -14,7 +14,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -31,7 +31,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -480,7 +480,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true }, "actions": [ diff --git a/reports/skillops/daily/2026-06-17.md b/reports/skillops/daily/2026-06-17.md index 47101e50..25bfffe0 100644 --- a/reports/skillops/daily/2026-06-17.md +++ b/reports/skillops/daily/2026-06-17.md @@ -1,6 +1,6 @@ # Daily SkillOps Report -Generated at: `2026-06-17T04:04:43Z` +Generated at: `2026-06-17T07:53:10Z` ## Summary @@ -14,7 +14,7 @@ Generated at: `2026-06-17T04:04:43Z` - local blueprint ready: `true` - public world-class ready: `false` - world-class pending: `4` -- release lock ready: `false` +- release lock ready: `true` - evidence consistency ok: `true` This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence. diff --git a/reports/skillops/weekly/2026-W25.json b/reports/skillops/weekly/2026-W25.json index 6653a1f9..44fc4b7f 100644 --- a/reports/skillops/weekly/2026-W25.json +++ b/reports/skillops/weekly/2026-W25.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T04:04:43Z", + "generated_at": "2026-06-17T07:53:10Z", "skill_dir": ".", "decision": "curator-review", "week_id": "2026-W25", @@ -15,7 +15,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -35,7 +35,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -288,7 +288,7 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": false, + "release_lock_ready": true, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4 diff --git a/reports/skillops/weekly/2026-W25.md b/reports/skillops/weekly/2026-W25.md index aaa81368..489bac8a 100644 --- a/reports/skillops/weekly/2026-W25.md +++ b/reports/skillops/weekly/2026-W25.md @@ -1,6 +1,6 @@ # Weekly SkillOps Curator Report -Generated at: `2026-06-17T04:04:43Z` +Generated at: `2026-06-17T07:53:10Z` Week: `2026-W25` ## Summary @@ -13,7 +13,7 @@ Week: `2026-W25` - top_score: `88` - skill_count: `12` - actionable_portfolio_issue_count: `0` -- release_lock_ready: `False` +- release_lock_ready: `True` - evidence_consistency_ok: `True` - public_world_class_ready: `False` - world_class_pending_count: `4` diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index 8c0c2c0f..fea69f16 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -349,7 +349,7 @@ "source_accepted": false, "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index c95ea28a..ef298e35 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -16,8 +16,8 @@ "missing_submission_count": 3, "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 12, - "source_blocked_count": 7, + "source_pass_count": 11, + "source_blocked_count": 8, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 1, "overclaim_guard_active": true, @@ -33,8 +33,8 @@ "missing_submission_count": 3, "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 12, - "source_blocked_count": 7, + "source_pass_count": 11, + "source_blocked_count": 8, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 1, "overclaim_guard_active": true, @@ -498,7 +498,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -535,7 +535,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -556,8 +556,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -573,8 +573,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index ecf6789e..d1e389f0 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -8,8 +8,8 @@ Generated at: `2026-06-17` - ready to claim world-class: `false` - entries: `4` - source accepted: `1` -- source checks: `12` pass / `19` total -- source blocked: `7` +- source checks: `11` pass / `19` total +- source blocked: `8` - accepted: `0` - pending: `4` - human pending: `1` @@ -30,7 +30,7 @@ This ledger records the current evidence state. It requires both passing source | `provider-holdout` | `pending` | `invalid-contract` | `external` | model-executed 10; token-observed 10 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | `human-adjudication` | `pending` | `missing` | `human` | 0/5 decisions; pending 5 | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | | `native-permission-enforcement` | `pending` | `missing` | `external` | native-enforced targets 0; installer-enforced targets 4 | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | -| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 1 | Install a real client against the native host and import production metadata-only events. | +| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. | ## Provider Holdout @@ -186,8 +186,8 @@ This ledger records the current evidence state. It requires both passing source - objective: Import production metadata-only events from a real external client into the local drift loop. - source status: `external_required` -- observed state: `{"external_source_events": 0, "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` -- source checks: `2` pass / `3` total +- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` +- source checks: `1` pass / `3` total - submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "ledger_counts_as_completion": false}` ### Provenance Requirements @@ -211,7 +211,7 @@ This ledger records the current evidence state. It requires both passing source | Check | Current | Expected | Status | | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | -| Adoption sample | `1` | `>0` | `pass` | +| Adoption sample | `0` | `>0` | `blocked` | | Raw content blocked | `False` | `false` | `pass` | ### Completion Assertions diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index f4ed0304..2ee34bbb 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -154,7 +154,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -325,7 +325,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index e46d8c01..2cf994c2 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -22,7 +22,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It | `provider-holdout` | `pass` | `external` | operator with provider credentials | model-executed 10; token-observed 10 | | `human-adjudication` | `human_required` | `human` | human reviewer | 0/5 decisions; pending 5 | | `native-permission-enforcement` | `external_required` | `external` | target client or installer integrator | native-enforced targets 0; installer-enforced targets 4 | -| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 1 | +| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 0 | ## Provider Holdout diff --git a/reports/world_class_evidence_preflight.html b/reports/world_class_evidence_preflight.html index 1cdcc21f..cb090424 100644 --- a/reports/world_class_evidence_preflight.html +++ b/reports/world_class_evidence_preflight.html @@ -68,7 +68,7 @@ Evidence Collection

World-Class Evidence Preflight

This operator view shows which external and human evidence is still blocked, which commands prepare editable submission drafts, and why preflight never counts as accepted evidence.

-
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source10/19
Repairs13/13
Phases2/2
+
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source11/19
Repairs12/12
Phases2/2
@@ -128,9 +128,9 @@
Phase
collect-source
-
Rows
9 / 9 blocked
-
Owners
Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator
-
Evidence
human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout
+
Rows
8 / 8 blocked
+
Owners
Browser/Chrome/IDE/provider client integrator, human reviewer, target client or installer integrator
+
Evidence
human-adjudication, native-client-telemetry, native-permission-enforcement
Next
Set reviewer_attestation only after choices are completed before opening the answer key.
Verify
python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
Counts
does not count as completion
@@ -147,13 +147,13 @@
Evidence
provider-holdout
Ledger
pending
-
Intake
awaiting-submission
-
Review
awaiting-submission
+
Intake
fix-submission
+
Review
fix-submission
Draft
evidence/world_class/submissions/provider-holdout.json

Next Action

-

Set OPENAI_API_KEY in the operator shell; never commit or print the value.

+

Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.

python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts
@@ -206,13 +206,13 @@
- env + env_any Provider credential
Current
not-set
Status
missing
-
Action
Set OPENAI_API_KEY in the operator shell; never commit or print the value.
+
Action
Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
@@ -224,7 +224,7 @@
Current
not-set
Status
optional
-
Action
Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.
+
Action
Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.
@@ -232,15 +232,15 @@

Source Checks

-
+
model_executed_count Provider model run
-
Current
0
+
Current
10
Expected
>0
-
Status
blocked
+
Status
pass
Action
Run provider-backed output-exec with real credentials.
@@ -258,15 +258,15 @@
-
+
token_observed_count Token usage observed
-
Current
0
+
Current
10
Expected
>0
-
Status
blocked
+
Status
pass
Action
Provider execution should return non-estimated token usage.
@@ -278,58 +278,24 @@
precheck - openai-api-key + provider-api-key
Priority
20
Phase
unblock-access
Owner
operator with provider credentials
Status
blocked
-
Reason
Required env precheck is missing.
-
Action
Set OPENAI_API_KEY in the operator shell; never commit or print the value.
+
Reason
Required env_any precheck is missing.
+
Action
Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
Verify
python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
Evidence
does not count as completion
- -
-
- source-check - model_executed_count -
-
-
Priority
40
-
Phase
collect-source
-
Owner
operator with provider credentials
-
Status
blocked
-
Reason
Current value 0 does not satisfy '>0'.
-
Action
Run provider-backed output-exec with real credentials.
-
Verify
python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
-
Evidence
does not count as completion
-
-
- -
-
- source-check - token_observed_count -
-
-
Priority
40
-
Phase
collect-source
-
Owner
operator with provider credentials
-
Status
blocked
-
Reason
Current value 0 does not satisfy '>0'.
-
Action
Provider execution should return non-estimated token usage.
-
Verify
python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
-
Evidence
does not count as completion
-
-

Runbook

-
  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+
  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
@@ -928,15 +894,15 @@ -
+
adoption_sample_count Adoption sample
-
Current
1
+
Current
0
Expected
>0
-
Status
pass
+
Status
blocked
Action
Telemetry must include adoption outcome evidence.
@@ -975,6 +941,23 @@
+
+
+ source-check + adoption_sample_count +
+
+
Priority
40
+
Phase
collect-source
+
Owner
Browser/Chrome/IDE/provider client integrator
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Telemetry must include adoption outcome evidence.
+
Verify
python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+
source-check diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index 80f304fe..ac62f375 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -14,14 +14,14 @@ "collection_ready_count": 1, "collection_blocked_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, - "repair_checklist_count": 13, - "repair_blocked_count": 13, + "source_pass_count": 11, + "source_blocked_count": 8, + "repair_checklist_count": 12, + "repair_blocked_count": 12, "repair_ready_count": 0, "repair_phase_counts": { "unblock-access": 4, - "collect-source": 9 + "collect-source": 8 }, "next_repair_action_id": "human-adjudication-precheck-human-reviewer", "next_repair_phase": "unblock-access", @@ -30,7 +30,7 @@ "repair_counts_as_completion": false, "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 13, + "phase_queue_row_count": 12, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -60,9 +60,9 @@ "owner": "operator with provider credentials", "status": "blocked", "ledger_status": "pending", - "intake_readiness": "awaiting-submission", - "review_state": "awaiting-submission", - "source_accepted": false, + "intake_readiness": "fix-submission", + "review_state": "fix-submission", + "source_accepted": true, "prechecks": [ { "evidence_key": "provider-holdout", @@ -90,16 +90,20 @@ }, { "evidence_key": "provider-holdout", - "key": "openai-api-key", + "key": "provider-api-key", "label": "Provider credential", - "kind": "env", + "kind": "env_any", "required": true, - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "secret_value_redacted": true, - "env": "OPENAI_API_KEY", + "env_any": [ + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY" + ], "status": "missing", "actual": "not-set", - "default": "" + "set_count": 0, + "set_envs": [] }, { "evidence_key": "provider-holdout", @@ -107,12 +111,12 @@ "label": "Provider model", "kind": "env", "required": false, - "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", "secret_value_redacted": false, "env": "YAO_OUTPUT_EVAL_MODEL", "status": "optional", "actual": "not-set", - "default": "gpt-4.1-mini" + "default": "provider-specific model, or pass --provider-model" } ], "source_checklist": [ @@ -121,9 +125,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -133,7 +137,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -141,13 +145,13 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." } ], - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", "commands": { @@ -192,58 +196,28 @@ "artifact_prefill_counts_as_evidence": false }, "runbook": [ - "Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.", - "export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}", - "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", "python3 scripts/yao.py skill-os2-audit . --generated-at ", "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" ], "repair_checklist": [ { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ], "phase_queue": [ @@ -260,77 +234,27 @@ "evidence_keys": [ "provider-holdout" ], - "next_action_id": "provider-holdout-precheck-openai-api-key", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false } ] - }, - { - "phase": "collect-source", - "label": "Collect source", - "priority": 40, - "status": "blocked", - "blocked_count": 2, - "row_count": 2, - "owners": [ - "operator with provider credentials" - ], - "evidence_keys": [ - "provider-holdout" - ], - "next_action_id": "provider-holdout-source-check-model_executed_count", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "counts_as_completion": false, - "rows": [ - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - } - ] } ] }, @@ -1076,8 +1000,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -1161,6 +1085,21 @@ "source": "prechecks", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1218,19 +1157,34 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", - "next_action": "Import at least one metadata-only event from a real client.", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1278,16 +1232,20 @@ }, { "evidence_key": "provider-holdout", - "key": "openai-api-key", + "key": "provider-api-key", "label": "Provider credential", - "kind": "env", + "kind": "env_any", "required": true, - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "secret_value_redacted": true, - "env": "OPENAI_API_KEY", + "env_any": [ + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY" + ], "status": "missing", "actual": "not-set", - "default": "" + "set_count": 0, + "set_envs": [] }, { "evidence_key": "provider-holdout", @@ -1295,12 +1253,12 @@ "label": "Provider model", "kind": "env", "required": false, - "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", "secret_value_redacted": false, "env": "YAO_OUTPUT_EVAL_MODEL", "status": "optional", "actual": "not-set", - "default": "gpt-4.1-mini" + "default": "provider-specific model, or pass --provider-model" }, { "evidence_key": "human-adjudication", @@ -1426,9 +1384,9 @@ "label": "Provider model run", "field": "model_executed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Run provider-backed output-exec with real credentials." }, { @@ -1438,7 +1396,7 @@ "expected": ">0", "actual": 10, "status": "pass", - "source_accepted": false, + "source_accepted": true, "next_action": "Provider execution should record timing metadata." }, { @@ -1446,9 +1404,9 @@ "label": "Token usage observed", "field": "token_observed_count", "expected": ">0", - "actual": 0, - "status": "blocked", - "source_accepted": false, + "actual": 10, + "status": "pass", + "source_accepted": true, "next_action": "Provider execution should return non-estimated token usage." }, { @@ -1596,8 +1554,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -1659,16 +1617,16 @@ "counts_as_completion": false }, { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false @@ -1748,6 +1706,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1777,36 +1750,6 @@ "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "source_checklist", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ], "phase_queue": [ @@ -1880,16 +1823,16 @@ "counts_as_completion": false }, { - "action_id": "provider-holdout-precheck-openai-api-key", + "action_id": "provider-holdout-precheck-provider-api-key", "evidence_key": "provider-holdout", "repair_type": "precheck", - "target": "openai-api-key", + "target": "provider-api-key", "phase": "unblock-access", "priority": 20, "owner": "operator with provider credentials", "status": "blocked", - "blocking_reason": "Required env precheck is missing.", - "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "prechecks", "counts_as_completion": false @@ -1901,19 +1844,17 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 9, - "row_count": 9, + "blocked_count": 8, + "row_count": 8, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", - "operator with provider credentials", "target client or installer integrator" ], "evidence_keys": [ "human-adjudication", "native-client-telemetry", - "native-permission-enforcement", - "provider-holdout" + "native-permission-enforcement" ], "next_action_id": "human-adjudication-source-check-blind_review_attested", "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", @@ -1995,6 +1936,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -2024,36 +1980,6 @@ "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "source": "source_checklist", "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-model_executed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "model_executed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Run provider-backed output-exec with real credentials.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false - }, - { - "action_id": "provider-holdout-source-check-token_observed_count", - "evidence_key": "provider-holdout", - "repair_type": "source-check", - "target": "token_observed_count", - "phase": "collect-source", - "priority": 40, - "owner": "operator with provider credentials", - "status": "blocked", - "blocking_reason": "Current value 0 does not satisfy '>0'.", - "next_action": "Provider execution should return non-estimated token usage.", - "verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", - "source": "source_checklist", - "counts_as_completion": false } ] } diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index d7343ba0..fb7579ea 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -10,10 +10,10 @@ Generated at: `2026-06-17` - credential value exposed: `false` - collection ready: `1` - collection blocked: `3` -- source checks: `10` pass / `19` total -- repair rows: `13` blocked / `13` total +- source checks: `11` pass / `19` total +- repair rows: `12` blocked / `12` total - phase queue: `2` blocked / `2` phases -- phase queue rows: `13` +- phase queue rows: `12` - next repair action: `human-adjudication-precheck-human-reviewer` - next repair owner: `human reviewer` - next phase: `unblock-access` @@ -51,13 +51,13 @@ Phase queue rows group the same repair checklist into operator execution phases. | Priority | Phase | Status | Rows | Owners | Evidence | Verify | Next action | | --- | --- | --- | --- | --- | --- | --- | --- | | `20` | `unblock-access` | `blocked` | 4 / 4 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Assign a real reviewer identity before claiming human adjudication. | -| `40` | `collect-source` | `blocked` | 9 / 9 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. | +| `40` | `collect-source` | `blocked` | 8 / 8 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. | ## Evidence Items | Evidence | Status | Intake | Review | Next action | | --- | --- | --- | --- | --- | -| `provider-holdout` | `blocked` | `awaiting-submission` | `awaiting-submission` | Set OPENAI_API_KEY in the operator shell; never commit or print the value. | +| `provider-holdout` | `blocked` | `fix-submission` | `fix-submission` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | | `human-adjudication` | `ready-for-human-review` | `awaiting-submission` | `awaiting-submission` | Assign a real reviewer identity before claiming human adjudication. | | `native-permission-enforcement` | `blocked` | `awaiting-submission` | `awaiting-submission` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | | `native-client-telemetry` | `blocked` | `awaiting-submission` | `awaiting-submission` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | @@ -71,16 +71,15 @@ Repair rows convert preflight and source blockers into a prioritized operator qu | `20` | `unblock-access` | human reviewer | `human-adjudication` | `precheck` | `human-reviewer` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Assign a real reviewer identity before claiming human adjudication. | | `20` | `unblock-access` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `precheck` | `external-client` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | | `20` | `unblock-access` | target client or installer integrator | `native-permission-enforcement` | `precheck` | `native-guard` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | -| `20` | `unblock-access` | operator with provider credentials | `provider-holdout` | `precheck` | `openai-api-key` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set OPENAI_API_KEY in the operator shell; never commit or print the value. | +| `20` | `unblock-access` | operator with provider credentials | `provider-holdout` | `precheck` | `provider-api-key` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | | `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `blind_review_attested` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. | | `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `judgment_count` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Every pair needs one valid human judgment. | | `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `pending_count` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Record a reviewer choice and reason for every pair. | | `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `ready_for_human_evidence` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints. | | `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `reviewer_metadata_present` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Record reviewer and reviewed_at before adjudication can count. | +| `40` | `collect-source` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `source-check` | `adoption_sample_count` | `blocked` | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Telemetry must include adoption outcome evidence. | | `40` | `collect-source` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `source-check` | `external_source_events` | `blocked` | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Import at least one metadata-only event from a real client. | | `40` | `collect-source` | target client or installer integrator | `native-permission-enforcement` | `source-check` | `native_enforcement_count` | `blocked` | `python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Collect real target-client or external runtime guard proof. | -| `40` | `collect-source` | operator with provider credentials | `provider-holdout` | `source-check` | `model_executed_count` | `blocked` | `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Run provider-backed output-exec with real credentials. | -| `40` | `collect-source` | operator with provider credentials | `provider-holdout` | `source-check` | `token_observed_count` | `blocked` | `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Provider execution should return non-estimated token usage. | ## Provider Holdout @@ -98,16 +97,16 @@ Repair rows convert preflight and source blockers into a prioritized operator qu | --- | --- | --- | --- | --- | | Output eval cases | `file` | `present` | `pass` | Keep output holdout cases available before provider execution. | | Provider runner | `file` | `present` | `pass` | Use the provider runner instead of the local command runner for model-backed evidence. | -| Provider credential | `env` | `not-set` | `missing` | Set OPENAI_API_KEY in the operator shell; never commit or print the value. | -| Provider model | `env` | `not-set` | `optional` | Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini. | +| Provider credential | `env_any` | `not-set` | `missing` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | +| Provider model | `env` | `not-set` | `optional` | Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider. | ### Source Checks | Check | Current | Expected | Status | Next action | | --- | --- | --- | --- | --- | -| Provider model run | `0` | `>0` | `blocked` | Run provider-backed output-exec with real credentials. | +| Provider model run | `10` | `>0` | `pass` | Run provider-backed output-exec with real credentials. | | Timing observed | `10` | `>0` | `pass` | Provider execution should record timing metadata. | -| Token usage observed | `0` | `>0` | `blocked` | Provider execution should return non-estimated token usage. | +| Token usage observed | `10` | `>0` | `pass` | Provider execution should return non-estimated token usage. | ## Human Adjudication @@ -192,7 +191,7 @@ Repair rows convert preflight and source blockers into a prioritized operator qu | Check | Current | Expected | Status | Next action | | --- | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | -| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. | +| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | | Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | ## Boundary diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html index 82ec145b..6731d2a6 100644 --- a/reports/world_class_operator_runbook.html +++ b/reports/world_class_operator_runbook.html @@ -68,11 +68,11 @@ Evidence Operations

World-Class Operator Runbook

A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

-
Pending4
Awaiting3
Ready0
Source12/19
Queue2/2
Steps6/6
Gate4/5
Blocked7
Invalid1
+
Pending4
Awaiting3
Ready0
Source11/19
Queue2/2
Steps6/6
Gate4/5
Blocked8
Invalid1

Fast Path

  1. Run the real external or human work for one evidence item.
  2. Generate and fill the matching submission draft.
  3. Validate intake and inspect the submission review queue.
  4. Refresh the ledger and run the claim guard before making any completion claim.

Coordination Plan

These steps tell the assistant what to run and where the user must supply real provider, reviewer, client, or telemetry input. Every row is operational guidance only and keeps counts_as_completion false.

StepEvidenceOwnerUserUser actionAssistant actionCommandPass condition
prepare-evidence-sessionallassistant + usertrueConfirm provider access, reviewer availability, target client path, and telemetry client path before collection starts.Run preflight and prepare submission drafts without accepting them as evidence.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifactsPreflight lists the same pending evidence keys and no credential values are printed.
collect-provider-holdoutprovider-holdoutassistant + operator with provider credentialstrueProvide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use.Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet.python3 scripts/yao.py output-exec . --provider-runner <openai|deepseek> --provider-model <model> --timeout-seconds 60reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0.
collect-human-adjudicationhuman-adjudicationhuman reviewer + assistanttrueOpen the blind review kit, choose winners for all pairs, add reviewer metadata and reasons, and keep the answer key hidden until decisions are saved.Generate the review kit, import decisions, validate integrity, and prepare the human evidence packet.python3 scripts/yao.py output-review-kit . && python3 scripts/yao.py output-review .reports/output_review_adjudication.json has pending_count == 0 and ready_for_human_evidence == true.
collect-native-permission-enforcementnative-permission-enforcementtarget client or installer integrator + assistanttrueSelect a real target client or external installer guard that can enforce declared capabilities instead of metadata-only fallback.Run runtime permission probes, package verification, install simulation, and prepare the native enforcement evidence packet.python3 scripts/yao.py runtime-permissions . --package-dir distreports/runtime_permission_probes.json has native_enforcement_count > 0 and failure_count == 0.
collect-native-client-telemetrynative-client-telemetryreal client integrator + assistanttrueInstall the native host manifest in a real Browser, Chrome, IDE, or provider client and trigger a metadata-only event.Generate native host assets, import the external event JSONL, refresh adoption drift, and prepare the telemetry evidence packet.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl --source externalreports/adoption_drift_report.json has source_types.external > 0 and adoption_sample_count > 0.
review-and-release-gateallassistant + ledger reviewertrueApprove only validated evidence packets and confirm the release wording after the claim guard passes.Run intake, submission review, ledger, claim guard, benchmark, evidence consistency, Review Studio, and CI before final publish.python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard . && make ci-testLedger ready_to_claim_world_class, benchmark public_claim_ready, claim guard violation_count == 0, Review Studio has no blockers, and CI passes.
-

Phase Queue

  • Unblock accessunblock-access: 4 / 4Assign a real reviewer identity before claiming human adjudication.
  • Collect sourcecollect-source: 7 / 7Set reviewer_attestation only after choices are completed before opening the answer key.
+

Phase Queue

  • Unblock accessunblock-access: 4 / 4Assign a real reviewer identity before claiming human adjudication.
  • Collect sourcecollect-source: 8 / 8Set reviewer_attestation only after choices are completed before opening the answer key.

Evidence Items

external · fix-submission

Provider Holdout

Submission exists but fails the ledger submission contract.

@@ -139,11 +139,11 @@
Owner
Browser/Chrome/IDE/provider client integrator
Ledger
pending
-
Blocked
1
+
Blocked
2
Queue
2
Submission
evidence/world_class/submissions/native-client-telemetry.json
-

Phase Queue

  • Unblock accessunblock-access: 1 / 1Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.
  • Collect sourcecollect-source: 1 / 1Import at least one metadata-only event from a real client.
+

Phase Queue

  • Unblock accessunblock-access: 1 / 1Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.
  • Collect sourcecollect-source: 2 / 2Telemetry must include adoption outcome evidence.

Source Runbook

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
@@ -151,8 +151,8 @@

Success Checks

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

Privacy

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
-

Next Source Actions

  • Import at least one metadata-only event from a real client.
-

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 1 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+

Next Source Actions

  • Import at least one metadata-only event from a real client.
  • Telemetry must include adoption outcome evidence.
+

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.

Release Gate

Decisionblocked-until-evidence-accepted
diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index f5a7dbe3..b61f0250 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -12,13 +12,13 @@ "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 12, - "source_blocked_count": 7, - "repair_checklist_count": 11, - "repair_blocked_count": 11, + "source_pass_count": 11, + "source_blocked_count": 8, + "repair_checklist_count": 12, + "repair_blocked_count": 12, "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 11, + "phase_queue_row_count": 12, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -912,7 +912,7 @@ "review_state": "awaiting-submission", "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "execution_runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -960,7 +960,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -981,8 +981,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -997,9 +997,10 @@ "next_action": "Telemetry must stay metadata-only." } ], - "blocked_source_check_count": 1, + "blocked_source_check_count": 2, "next_source_actions": [ - "Import at least one metadata-only event from a real client." + "Import at least one metadata-only event from a real client.", + "Telemetry must include adoption outcome evidence." ], "repair_checklist": [ { @@ -1017,6 +1018,21 @@ "source": "prechecks", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1033,7 +1049,7 @@ "counts_as_completion": false } ], - "repair_blocked_count": 2, + "repair_blocked_count": 3, "repair_counts_as_completion": false, "phase_queue": [ { @@ -1076,19 +1092,34 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", - "next_action": "Import at least one metadata-only event from a real client.", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1440,6 +1471,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1563,8 +1609,8 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 7, - "row_count": 7, + "blocked_count": 8, + "row_count": 8, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", @@ -1655,6 +1701,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index ed8ac4bc..92df3ed8 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -12,7 +12,7 @@ Generated at: `2026-06-17` - awaiting submission: `3` - ready for ledger review: `0` - phase queue: `2` blocked / `2` phases -- phase queue rows: `11` +- phase queue rows: `12` - phase queue counts as completion: `false` - coordination steps: `6` user-required / `6` total - coordination pending keys: `human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout` @@ -46,7 +46,7 @@ This runbook coordinates evidence collection only. It does not accept submission | Phase | Status | Rows | Blocked | Owners | Next action | Verify | | --- | --- | ---: | ---: | --- | --- | --- | | `unblock-access` | `blocked` | `4` | `4` | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | Assign a real reviewer identity before claiming human adjudication. | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | -| `collect-source` | `blocked` | `7` | `7` | Browser/Chrome/IDE/provider client integrator, human reviewer, target client or installer integrator | Set reviewer_attestation only after choices are completed before opening the answer key. | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | +| `collect-source` | `blocked` | `8` | `8` | Browser/Chrome/IDE/provider client integrator, human reviewer, target client or installer integrator | Set reviewer_attestation only after choices are completed before opening the answer key. | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | ## Evidence Items @@ -55,7 +55,7 @@ This runbook coordinates evidence collection only. It does not accept submission | `provider-holdout` | `pending` | `fix-submission` | `fix-submission` | `0` | none | operator with provider credentials | | `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | `5` | Record a reviewer choice and reason for every pair. | human reviewer | | `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Collect real target-client or external runtime guard proof. | target client or installer integrator | -| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | +| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | ## Provider Holdout @@ -313,8 +313,8 @@ This runbook coordinates evidence collection only. It does not accept submission - objective: Import production metadata-only events from a real external client into the local drift loop. - blocking reason: No evidence packet has been submitted for review. -- blocked source checks: `1` -- repair rows: `2` blocked +- blocked source checks: `2` +- repair rows: `3` blocked - phase queue: `2` blocked phases - submission: `evidence/world_class/submissions/native-client-telemetry.json` - template: `evidence/world_class/templates/native-client-telemetry.intake.json` @@ -324,7 +324,7 @@ This runbook coordinates evidence collection only. It does not accept submission | Phase | Status | Rows | Blocked | Next action | | --- | --- | ---: | ---: | --- | | `unblock-access` | `blocked` | `1` | `1` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | -| `collect-source` | `blocked` | `1` | `1` | Import at least one metadata-only event from a real client. | +| `collect-source` | `blocked` | `2` | `2` | Telemetry must include adoption outcome evidence. | ### Source Runbook @@ -375,13 +375,14 @@ This runbook coordinates evidence collection only. It does not accept submission ### Next Source Actions - Import at least one metadata-only event from a real client. +- Telemetry must include adoption outcome evidence. ### Source Evidence Snapshot | Check | Current | Expected | Status | Next action | | --- | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | -| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. | +| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | | Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | ## Release Gate diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index 6e690725..0d44ec77 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -14,8 +14,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 12, - "source_blocked_count": 7, + "source_pass_count": 11, + "source_blocked_count": 8, "review_counts_submission_as_completion": false, "summary": { "review_item_count": 4, @@ -27,8 +27,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 1, "source_check_count": 19, - "source_pass_count": 12, - "source_blocked_count": 7, + "source_pass_count": 11, + "source_blocked_count": 8, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "fix-submissions" @@ -384,7 +384,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -405,8 +405,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -422,8 +422,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 8bb2b01d..07503ec9 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -157,7 +157,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Source Checks - External events: 0 / >0 => blocked -- Adoption sample: 1 / >0 => pass +- Adoption sample: 0 / >0 => blocked - Raw content blocked: False / false => pass #### Completion Assertions