From 136654cda911836efd177294a17efc4e5bf08ae9 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Wed, 17 Jun 2026 05:01:41 +0800 Subject: [PATCH] Refresh full phase queue evidence bundle --- reports/adoption_drift_report.json | 28 +- reports/adoption_drift_report.md | 10 +- reports/benchmark_reproducibility.json | 88 ++--- reports/benchmark_reproducibility.md | 32 +- reports/context_budget.json | 4 +- reports/context_budget_summary.json | 2 +- reports/evidence_consistency.json | 160 +++++---- reports/output_execution_runs.json | 20 +- reports/output_execution_runs.md | 20 +- reports/review-studio.html | 16 +- reports/review-studio.json | 377 ++++++++++++-------- reports/review-viewer.json | 45 +-- reports/skill-interpretation.html | 2 +- reports/skill-interpretation.json | 45 +-- reports/skill-overview.html | 2 +- reports/skill-overview.json | 45 +-- reports/world_class_evidence_intake.json | 2 +- reports/world_class_evidence_ledger.json | 20 +- reports/world_class_evidence_ledger.md | 12 +- reports/world_class_evidence_plan.json | 4 +- reports/world_class_evidence_plan.md | 2 +- reports/world_class_evidence_preflight.html | 27 +- reports/world_class_evidence_preflight.json | 92 ++++- reports/world_class_evidence_preflight.md | 11 +- reports/world_class_operator_runbook.html | 8 +- reports/world_class_operator_runbook.json | 17 +- reports/world_class_operator_runbook.md | 7 +- reports/world_class_submission_review.json | 18 +- reports/world_class_submission_review.md | 2 +- 29 files changed, 643 insertions(+), 475 deletions(-) diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 37100b8..a423e3d 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16T20:58:58Z", + "generated_at": "2026-06-17", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -25,14 +25,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -41,7 +41,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -53,31 +53,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "skill_activation", + "event": "review_event", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } diff --git a/reports/adoption_drift_report.md b/reports/adoption_drift_report.md index 206372c..91b3198 100644 --- a/reports/adoption_drift_report.md +++ b/reports/adoption_drift_report.md @@ -5,9 +5,9 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Summary - Events: `1` -- Adoption samples: `1` -- Activation events: `1` -- Adoption rate: `100.0` +- Adoption samples: `0` +- Activation events: `0` +- Adoption rate: `0` - Missed trigger signals: `0` - Bad output signals: `0` - Script error signals: `0` @@ -25,7 +25,7 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, | Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| `yao-meta-skill` | 1 | 1 | 1 | 0 | 0 | 0 | 100.0 | +| `yao-meta-skill` | 1 | 0 | 0 | 0 | 0 | 0 | 0 | ## Next Iteration Candidates @@ -33,4 +33,4 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Recent Metadata Events -- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` source=`manual` command=`unknown` activation=`explicit` outcome=`accepted` failure=`none` +- `2026-06-13T12:00:00Z` `yao-meta-skill` event=`review_event` source=`manual` command=`unknown` activation=`manual` outcome=`reviewed` failure=`none` diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 8d86f98..6af364d 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,43 +3,43 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "commit": "28c48ec248807020a68b50e012d4419edafbb418", "git_status": { "available": true, "dirty": true, - "changed_file_count": 17, + "changed_file_count": 18, "generated_dirty": true, - "generated_changed_file_count": 17, + "generated_changed_file_count": 18, "source_dirty": false, "source_changed_file_count": 0, "sample": [ - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", " M reports/context_budget.json", - " M reports/evidence_consistency.json", + " M reports/context_budget_summary.json", " M reports/output_execution_runs.json", " M reports/output_execution_runs.md", - " M reports/review-studio.html", - " M reports/review-studio.json", - " M reports/review-viewer.json", - " M reports/skill-interpretation.json" + " M reports/world_class_evidence_intake.json", + " M reports/world_class_evidence_ledger.json", + " M reports/world_class_evidence_ledger.md", + " M reports/world_class_evidence_plan.json", + " M reports/world_class_evidence_plan.md", + " M reports/world_class_evidence_preflight.json" ], "source_sample": [], "generated_sample": [ - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", " M reports/context_budget.json", - " M reports/evidence_consistency.json", + " M reports/context_budget_summary.json", " M reports/output_execution_runs.json", " M reports/output_execution_runs.md", - " M reports/review-studio.html", - " M reports/review-studio.json", - " M reports/review-viewer.json", - " M reports/skill-interpretation.json" + " M reports/world_class_evidence_intake.json", + " M reports/world_class_evidence_ledger.json", + " M reports/world_class_evidence_ledger.md", + " M reports/world_class_evidence_plan.json", + " M reports/world_class_evidence_plan.md", + " M reports/world_class_evidence_preflight.json" ], "generated_dirty_prefixes": [ "dist/", @@ -57,7 +57,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf", + "evidence_bundle_sha256": "2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403", "source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -74,16 +74,16 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4, "working_tree_dirty": true, - "changed_file_count": 17, + "changed_file_count": 18, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 17 + "generated_changed_file_count": 18 }, "public_claim": { "ready": false, @@ -92,16 +92,16 @@ "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (10/19 pass, 9 blocked)" + "world-class source checks are not all accepted (9/19 pass, 10 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { "ready": true, - "commit": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "commit": "28c48ec248807020a68b50e012d4419edafbb418", "status_scope": "generation-time status before this report is written", "source_changed_file_count": 0, - "generated_changed_file_count": 17, + "generated_changed_file_count": 18, "reason": "only generated evidence artifacts were dirty at generation time" }, "evidence_bundle": { @@ -110,7 +110,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf" + "sha256": "2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -184,7 +184,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "6edee9440d276bd62d36deee7a5a2c45ce2e7c6817a0e517f0339b266e4055aa" + "sha256": "26c5abca0e771fbdbb1e27dd4fe4868d8b379a92df3d40517932ed5346c5100c" }, { "label": "blind_review", @@ -261,56 +261,56 @@ "path": "reports/world_class_evidence_plan.json", "exists": true, "bytes": 22784, - "sha256": "d0ab417742950ff5a5f3407f068829217da5c997a6a7c5a5ee59441cb4abfb7f" + "sha256": "2beb888d7992fc72073251632a6e85c4075278f9bcea946a46a7e11f8f43e0ca" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 26016, - "sha256": "fc4c3e398ad0de110ae5f37e461d03712716552ce85b44e41e09f4d343a17f0b" + "bytes": 26019, + "sha256": "22c85613d7bef62c4d0a4c94b77a8596090dc55c40e1663370d16d903e0b1ffb" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, "bytes": 20646, - "sha256": "1a939bdd1f4a3ee30e05f0b286145f0f3320ba9f1935749806a581737155774a" + "sha256": "d3f44570e605e6e1b1557889ccca18efbbd8acc1d9a9c638f59961b3dc3e9026" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, "bytes": 97462, - "sha256": "d6c13c020987c1b395d8e78cc409fec0784f6746ff77f0ef148abf105b9d6045" + "sha256": "74c0cb843c38aa63434a1a9109e7bccccc60e675b8025183dd8a1b66a9def7b4" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 17299, - "sha256": "b256c40e75283556fc2ab1a0ff3a19834e82d54d4c7a3e7464806c01aab69db7" + "bytes": 17302, + "sha256": "96624a5b7f79aa0ac43ce60d30486290b9a91f0735254e31329f02e1ee291050" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 28871, - "sha256": "3bf99aca0798e86e13a5364aaf8462f23cdb472770671f4bef41ac50abf65c32" + "bytes": 28935, + "sha256": "a7b7809d3b423bf647a528c3b0f6991f2ea3874afdd93a4d6ba74eb189131f2a" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 17521, - "sha256": "69b7ba66e5e2710438d168e2e9e9c5a6187c9f8dadc6a1e7a0f60e4dc222d679" + "bytes": 17576, + "sha256": "d72777b421b57f0cf8f31b4d048cf92512e49216d0b0c05cec1686d4aea638ce" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 23958, - "sha256": "46b435f98d8a91501580fa1426a8ab0d0f57b3bc92ecd17f42332e655fd2bd85" + "bytes": 24019, + "sha256": "73d8ef2898f25ae899a618b723f40a6907646c303effa21cbde6156d01a95fc7" }, { "label": "world_class_claim_guard", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 4d6c72a..8aa9470 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,11 +1,11 @@ # Benchmark Reproducibility Generated at: `2026-06-17` -Commit: `9eaa8f6cec4ba83c07ded0275686252576bee6c3` +Commit: `28c48ec248807020a68b50e012d4419edafbb418` Working tree dirty at generation: `true` Source tree dirty at generation: `false` Generated evidence dirty at generation: `true` -Evidence bundle SHA256: `98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf` +Evidence bundle SHA256: `2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403` ## Summary @@ -22,12 +22,12 @@ Evidence bundle SHA256: `98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b45 - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- world-class source checks: `10` pass / `19` total; `9` blocked +- world-class source checks: `9` pass / `19` total; `10` blocked - public claim ready: `false` - public claim blockers: `4` -- changed files at generation: `17` +- changed files at generation: `18` - source changed files at generation: `0` -- generated changed files at generation: `17` +- generated changed files at generation: `18` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -42,7 +42,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | -| world-class source checks are not all accepted (10/19 pass, 9 blocked) | +| world-class source checks are not all accepted (9/19 pass, 10 blocked) | ## Release Lock @@ -54,7 +54,7 @@ This report proves local benchmark reproducibility only. It keeps external provi - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf` +- sha256: `2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403` ## Methodology Sections @@ -76,7 +76,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `6edee9440d27` | +| output_execution | `reports/output_execution_runs.json` | present | `26c5abca0e77` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `91fd88dd9b0f` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | @@ -87,14 +87,14 @@ This report proves local benchmark reproducibility only. It keeps external provi | package_verification | `reports/package_verification.json` | present | `7aa6c25e8154` | | install_simulation | `reports/install_simulation.json` | present | `d2bea19ea46f` | | skill_os2_audit | `reports/skill_os2_audit.json` | present | `ad57230481ed` | -| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `d0ab41774295` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `fc4c3e398ad0` | -| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `1a939bdd1f4a` | -| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `d6c13c020987` | -| world_class_submission_review | `reports/world_class_submission_review.json` | present | `b256c40e7528` | -| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `3bf99aca0798` | -| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `69b7ba66e5e2` | -| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `46b435f98d8a` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `2beb888d7992` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `22c85613d7be` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `d3f44570e605` | +| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `74c0cb843c38` | +| world_class_submission_review | `reports/world_class_submission_review.json` | present | `96624a5b7f79` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `a7b7809d3b42` | +| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `d72777b421b5` | +| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `73d8ef2898f2` | | world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `abe7f7d60c00` | ## Reproduction Commands diff --git a/reports/context_budget.json b/reports/context_budget.json index 22efeda..bdd81de 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,9 +6,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1121236, + "other_text_tokens": 1121243, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1122033, + "estimated_total_text_tokens": 1122040, "deferred_resource_tokens": 512782, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index b4fc88d..9105420 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-03-31", + "generated_at": "2026-06-17", "targets": [ { "label": "root", diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 41ca9bd..1644d62 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -222,7 +222,7 @@ "actual": { "available": true, "clean": false, - "changed_file_count": 14 + "changed_file_count": 28 }, "paths": [ "reports/benchmark_reproducibility.json" @@ -263,8 +263,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", - "actual": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "expected": "28c48ec248807020a68b50e012d4419edafbb418", + "actual": "28c48ec248807020a68b50e012d4419edafbb418", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -283,8 +283,8 @@ "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4 }, @@ -296,8 +296,8 @@ "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4 }, @@ -313,13 +313,13 @@ "status": "pass", "expected": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, - "adoption_rate": 100.0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "source_types": { "manual": 1 @@ -327,13 +327,13 @@ }, "actual": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, - "adoption_rate": 100.0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "source_types": { "manual": 1 @@ -356,8 +356,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -368,8 +368,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -389,7 +389,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 19, - "source_pass_count": 10 + "source_pass_count": 9 }, "actual": { "ready": false, @@ -397,7 +397,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 19, - "source_pass_count": 10 + "source_pass_count": 9 }, "paths": [ "reports/world_class_evidence_ledger.json", @@ -409,8 +409,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", - "actual": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "expected": "28c48ec248807020a68b50e012d4419edafbb418", + "actual": "28c48ec248807020a68b50e012d4419edafbb418", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -429,8 +429,8 @@ "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4 }, @@ -442,8 +442,8 @@ "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4 }, @@ -459,13 +459,13 @@ "status": "pass", "expected": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, - "adoption_rate": 100.0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "source_types": { "manual": 1 @@ -473,13 +473,13 @@ }, "actual": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, - "adoption_rate": 100.0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "source_types": { "manual": 1 @@ -502,8 +502,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -514,8 +514,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "decision": "evidence-pending" }, @@ -535,7 +535,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 19, - "source_pass_count": 10 + "source_pass_count": 9 }, "actual": { "ready": false, @@ -543,7 +543,7 @@ "pending_count": 4, "accepted_count": 0, "source_check_count": 19, - "source_pass_count": 10 + "source_pass_count": 9 }, "paths": [ "reports/world_class_evidence_ledger.json", @@ -1342,7 +1342,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1402,7 +1402,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -1416,7 +1417,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1476,7 +1477,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -1826,15 +1828,15 @@ "expected": { "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false }, "actual": { "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false }, "paths": [ @@ -1850,8 +1852,8 @@ "expected": { "pending_count": 4, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "preflight_counts_as_evidence": false, "credential_value_exposed": false @@ -1859,8 +1861,8 @@ "actual": { "pending_count": 4, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "preflight_counts_as_evidence": false, "credential_value_exposed": false @@ -2037,7 +2039,7 @@ "summary": { "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 13, + "phase_queue_row_count": 14, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2070,8 +2072,8 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 9, - "row_count": 9, + "blocked_count": 10, + "row_count": 10, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", @@ -2213,15 +2215,15 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false } @@ -2352,15 +2354,15 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false } @@ -2371,7 +2373,7 @@ "summary": { "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 13, + "phase_queue_row_count": 14, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -2404,8 +2406,8 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 9, - "row_count": 9, + "blocked_count": 10, + "row_count": 10, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", @@ -2547,15 +2549,15 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false } @@ -2685,15 +2687,15 @@ "phase": "collect-source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false } @@ -2792,8 +2794,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "plan_keys": [ "human-adjudication", "native-client-telemetry", @@ -2941,8 +2943,8 @@ "human_pending_count": 1, "external_pending_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "plan_keys": [ "human-adjudication", "native-client-telemetry", diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index f4927be..45cf17c 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.72, + "duration_ms": 26.66, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.31, + "duration_ms": 26.23, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.59, + "duration_ms": 25.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.92, + "duration_ms": 28.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.16, + "duration_ms": 28.48, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.43, + "duration_ms": 26.24, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.62, + "duration_ms": 25.96, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.78, + "duration_ms": 26.05, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.76, + "duration_ms": 25.84, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.99, + "duration_ms": 25.86, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 19e65aa..faee1a6 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 31.72 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 28.31 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 28.59 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 27.92 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 28.16 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.43 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.62 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.78 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 27.76 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.99 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 26.66 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 26.23 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 25.95 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.04 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 28.48 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.24 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.96 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.05 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.84 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.86 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/review-studio.html b/reports/review-studio.html index 68ca61b..34d5504 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -777,7 +777,7 @@

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 512782/120000; top deferred scripts 451479; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 152 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 231 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

228 Python files; 0 hotspots; 0 watchlist files; 3 early watch files; 0 blockers; largest 696 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 512782/120000; top deferred scripts 451479; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 152 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 231 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

228 Python files; 0 hotspots; 0 watchlist files; 3 early watch files; 0 blockers; largest 696 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 9/19 pass; 10 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
@@ -787,12 +787,12 @@

阻断事项

无。

-

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true
+

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 9/19 pass; 10 blocked; overclaim guard true

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md + reports/output_review_decisions.json
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
修复
3 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: operator with provider credentialsevidence: provider-holdoutSet OPENAI_API_KEY in the operator shell; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source2 / 2 blockedowners: operator with provider credentialsevidence: provider-holdoutRun provider-backed output-exec with real credentials.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckopenai-api-keyoperator with provider credentialsRequired env precheck is missing.Set OPENAI_API_KEY in the operator shell; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkmodel_executed_countoperator with provider credentialsCurrent value 0 does not satisfy '>0'.Run provider-backed output-exec with real credentials.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checktoken_observed_countoperator with provider credentialsCurrent value 0 does not satisfy '>0'.Provider execution should return non-estimated token usage.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  2. export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  3. python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
5 blocked / 5 pass
修复
6 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: human reviewerevidence: human-adjudicationAssign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source5 / 5 blockedowners: human reviewerevidence: human-adjudicationSet reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckhuman-reviewerhuman reviewerRequired human precheck is human-required.Assign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkblind_review_attestedhuman reviewerCurrent value False does not satisfy 'true'.Set reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkjudgment_counthuman reviewerCurrent value 0 does not satisfy '==pair_count'.Every pair needs one valid human judgment.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkpending_counthuman reviewerCurrent value 5 does not satisfy '==0'.Record a reviewer choice and reason for every pair.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkready_for_human_evidencehuman reviewerCurrent value False does not satisfy 'true'.Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkreviewer_metadata_presenthuman reviewerCurrent value False does not satisfy 'true'.Record reviewer and reviewed_at before adjudication can count.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
修复
2 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementAttach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementCollect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · prechecknative-guardtarget client or installer integratorRequired external precheck is external-required.Attach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checknative_enforcement_counttarget client or installer integratorCurrent value 0 does not satisfy '>0'.Collect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 1

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
1 blocked / 2 pass
修复
2 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryInstall a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source1 / 1 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryImport at least one metadata-only event from a real client.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckexternal-clientBrowser/Chrome/IDE/provider client integratorRequired external precheck is external-required.Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkexternal_source_eventsBrowser/Chrome/IDE/provider client integratorCurrent value 0 does not satisfy '>0'.Import at least one metadata-only event from a real client.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md + reports/output_review_decisions.json
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
修复
3 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: operator with provider credentialsevidence: provider-holdoutSet OPENAI_API_KEY in the operator shell; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source2 / 2 blockedowners: operator with provider credentialsevidence: provider-holdoutRun provider-backed output-exec with real credentials.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckopenai-api-keyoperator with provider credentialsRequired env precheck is missing.Set OPENAI_API_KEY in the operator shell; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkmodel_executed_countoperator with provider credentialsCurrent value 0 does not satisfy '>0'.Run provider-backed output-exec with real credentials.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checktoken_observed_countoperator with provider credentialsCurrent value 0 does not satisfy '>0'.Provider execution should return non-estimated token usage.python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  2. export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  3. python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
5 blocked / 5 pass
修复
6 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: human reviewerevidence: human-adjudicationAssign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source5 / 5 blockedowners: human reviewerevidence: human-adjudicationSet reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckhuman-reviewerhuman reviewerRequired human precheck is human-required.Assign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkblind_review_attestedhuman reviewerCurrent value False does not satisfy 'true'.Set reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkjudgment_counthuman reviewerCurrent value 0 does not satisfy '==pair_count'.Every pair needs one valid human judgment.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkpending_counthuman reviewerCurrent value 5 does not satisfy '==0'.Record a reviewer choice and reason for every pair.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkready_for_human_evidencehuman reviewerCurrent value False does not satisfy 'true'.Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkreviewer_metadata_presenthuman reviewerCurrent value False does not satisfy 'true'.Record reviewer and reviewed_at before adjudication can count.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
修复
2 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementAttach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementCollect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · prechecknative-guardtarget client or installer integratorRequired external precheck is external-required.Attach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checknative_enforcement_counttarget client or installer integratorCurrent value 0 does not satisfy '>0'.Collect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
修复
3 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryInstall a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source2 / 2 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryTelemetry must include adoption outcome evidence.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckexternal-clientBrowser/Chrome/IDE/provider client integratorRequired external precheck is external-required.Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkadoption_sample_countBrowser/Chrome/IDE/provider client integratorCurrent value 0 does not satisfy '>0'.Telemetry must include adoption outcome evidence.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkexternal_source_eventsBrowser/Chrome/IDE/provider client integratorCurrent value 0 does not satisfy '>0'.Import at least one metadata-only event from a real client.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -874,8 +874,8 @@
-

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

-

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

+

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
@@ -899,7 +899,7 @@

世界证据

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

-
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; reviewer_metadata_present: False; reason_required: True; blind_review_attested: False; raw_content_excluded_attested: True; reviewer_reason_required_attested: True; ready_for_human_evidence: False; raw_content_allowed: False; raw_content_path_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Reason requiredreason_required: True / trueKeep reason mandatory for every imported or direct reviewer decision.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Raw content attestedraw_content_excluded_attested: True / trueAttest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
  • Raw content blockedraw_content_allowed: False / falseAdjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 1
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 1; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 1 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; reviewer_metadata_present: False; reason_required: True; blind_review_attested: False; raw_content_excluded_attested: True; reviewer_reason_required_attested: True; ready_for_human_evidence: False; raw_content_allowed: False; raw_content_path_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Reason requiredreason_required: True / trueKeep reason mandatory for every imported or direct reviewer decision.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Raw content attestedraw_content_excluded_attested: True / trueAttest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
  • Raw content blockedraw_content_allowed: False / falseAdjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
@@ -915,11 +915,11 @@

公开声明

本地复现
是
发布锁
是
可公开声明
否
声明阻断
4
Provider 证据
否
人审完成
否
世界级就绪
否
-

声明阻断

  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (10/19 pass, 9 blocked)
+

声明阻断

  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (9/19 pass, 10 blocked)
-

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true

+

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 9/19 pass; 10 blocked; overclaim guard true

证据台账

Ledger Entry Count
4
Accepted Count
0
待审
4
Human Pending Count
1
External Pending Count
3
Overclaim Guard Active
是
Ready To Claim World Class
否
diff --git a/reports/review-studio.json b/reports/review-studio.json index 67400de..299f0cd 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -108,7 +108,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", + "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", "evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", "link": "adoption_drift_report.md" }, @@ -124,7 +124,7 @@ "key": "world-class-evidence", "label": "世界证据", "status": "warn", - "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 9/19 pass; 10 blocked; overclaim guard true", "evidence": "reports/world_class_evidence_ledger.json", "link": "world_class_evidence_ledger.md" }, @@ -212,7 +212,7 @@ "key": "world-class-evidence", "label": "世界证据", "status": "warn", - "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 10/19 pass; 9 blocked; overclaim guard true", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 9/19 pass; 10 blocked; overclaim guard true", "evidence": "reports/world_class_evidence_ledger.json", "link": "world_class_evidence_ledger.md" } @@ -1142,12 +1142,12 @@ "category": "external", "status": "pending", "readiness": "awaiting-submission", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "next_action": "Install a real client against the native host and import production metadata-only events.", "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "blocked_checks": [ { "label": "External events", @@ -1156,6 +1156,14 @@ "expected": ">0", "status": "blocked", "next_action": "Import at least one metadata-only event from a real client." + }, + { + "label": "Adoption sample", + "field": "adoption_sample_count", + "actual": 0, + "expected": ">0", + "status": "blocked", + "next_action": "Telemetry must include adoption outcome evidence." } ], "repair_rows": [ @@ -1172,6 +1180,19 @@ "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "repair_type": "source-check", @@ -1186,7 +1207,7 @@ "counts_as_completion": false } ], - "repair_blocked_count": 2, + "repair_blocked_count": 3, "repair_counts_as_completion": false, "phase_queue": [ { @@ -1212,16 +1233,16 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", - "next_action": "Import at least one metadata-only event from a real client.", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false } @@ -1770,7 +1791,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1830,7 +1851,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -2367,7 +2389,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf", + "evidence_bundle_sha256": "2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403", "source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -2384,18 +2406,18 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4, "working_tree_dirty": true, - "changed_file_count": 17, + "changed_file_count": 18, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 17 + "generated_changed_file_count": 18 }, - "commit": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "commit": "28c48ec248807020a68b50e012d4419edafbb418", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", @@ -2687,14 +2709,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -2703,7 +2725,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -2978,7 +3000,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -3031,8 +3053,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -3458,7 +3480,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -3495,7 +3517,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -3516,8 +3538,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -3533,8 +3555,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -5184,7 +5206,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 37.66, + "duration_ms": 26.66, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5212,7 +5234,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.99, + "duration_ms": 26.23, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5235,7 +5257,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.31, + "duration_ms": 25.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5263,7 +5285,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.12, + "duration_ms": 28.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5286,7 +5308,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.69, + "duration_ms": 28.48, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5314,7 +5336,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.85, + "duration_ms": 26.24, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5337,7 +5359,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.37, + "duration_ms": 25.96, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5364,7 +5386,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.98, + "duration_ms": 26.05, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5387,7 +5409,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.23, + "duration_ms": 25.84, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5416,7 +5438,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.43, + "duration_ms": 25.86, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -6197,43 +6219,43 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "commit": "28c48ec248807020a68b50e012d4419edafbb418", "git_status": { "available": true, "dirty": true, - "changed_file_count": 17, + "changed_file_count": 18, "generated_dirty": true, - "generated_changed_file_count": 17, + "generated_changed_file_count": 18, "source_dirty": false, "source_changed_file_count": 0, "sample": [ - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", " M reports/context_budget.json", - " M reports/evidence_consistency.json", + " M reports/context_budget_summary.json", " M reports/output_execution_runs.json", " M reports/output_execution_runs.md", - " M reports/review-studio.html", - " M reports/review-studio.json", - " M reports/review-viewer.json", - " M reports/skill-interpretation.json" + " M reports/world_class_evidence_intake.json", + " M reports/world_class_evidence_ledger.json", + " M reports/world_class_evidence_ledger.md", + " M reports/world_class_evidence_plan.json", + " M reports/world_class_evidence_plan.md", + " M reports/world_class_evidence_preflight.json" ], "source_sample": [], "generated_sample": [ - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", " M reports/context_budget.json", - " M reports/evidence_consistency.json", + " M reports/context_budget_summary.json", " M reports/output_execution_runs.json", " M reports/output_execution_runs.md", - " M reports/review-studio.html", - " M reports/review-studio.json", - " M reports/review-viewer.json", - " M reports/skill-interpretation.json" + " M reports/world_class_evidence_intake.json", + " M reports/world_class_evidence_ledger.json", + " M reports/world_class_evidence_ledger.md", + " M reports/world_class_evidence_plan.json", + " M reports/world_class_evidence_plan.md", + " M reports/world_class_evidence_preflight.json" ], "generated_dirty_prefixes": [ "dist/", @@ -6251,7 +6273,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf", + "evidence_bundle_sha256": "2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403", "source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -6268,16 +6290,16 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4, "working_tree_dirty": true, - "changed_file_count": 17, + "changed_file_count": 18, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 17 + "generated_changed_file_count": 18 }, "public_claim": { "ready": false, @@ -6286,16 +6308,16 @@ "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (10/19 pass, 9 blocked)" + "world-class source checks are not all accepted (9/19 pass, 10 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { "ready": true, - "commit": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "commit": "28c48ec248807020a68b50e012d4419edafbb418", "status_scope": "generation-time status before this report is written", "source_changed_file_count": 0, - "generated_changed_file_count": 17, + "generated_changed_file_count": 18, "reason": "only generated evidence artifacts were dirty at generation time" }, "evidence_bundle": { @@ -6304,7 +6326,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf" + "sha256": "2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -6378,7 +6400,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "6edee9440d276bd62d36deee7a5a2c45ce2e7c6817a0e517f0339b266e4055aa" + "sha256": "26c5abca0e771fbdbb1e27dd4fe4868d8b379a92df3d40517932ed5346c5100c" }, { "label": "blind_review", @@ -6455,56 +6477,56 @@ "path": "reports/world_class_evidence_plan.json", "exists": true, "bytes": 22784, - "sha256": "d0ab417742950ff5a5f3407f068829217da5c997a6a7c5a5ee59441cb4abfb7f" + "sha256": "2beb888d7992fc72073251632a6e85c4075278f9bcea946a46a7e11f8f43e0ca" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 26016, - "sha256": "fc4c3e398ad0de110ae5f37e461d03712716552ce85b44e41e09f4d343a17f0b" + "bytes": 26019, + "sha256": "22c85613d7bef62c4d0a4c94b77a8596090dc55c40e1663370d16d903e0b1ffb" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, "bytes": 20646, - "sha256": "1a939bdd1f4a3ee30e05f0b286145f0f3320ba9f1935749806a581737155774a" + "sha256": "d3f44570e605e6e1b1557889ccca18efbbd8acc1d9a9c638f59961b3dc3e9026" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, "bytes": 97462, - "sha256": "d6c13c020987c1b395d8e78cc409fec0784f6746ff77f0ef148abf105b9d6045" + "sha256": "74c0cb843c38aa63434a1a9109e7bccccc60e675b8025183dd8a1b66a9def7b4" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 17299, - "sha256": "b256c40e75283556fc2ab1a0ff3a19834e82d54d4c7a3e7464806c01aab69db7" + "bytes": 17302, + "sha256": "96624a5b7f79aa0ac43ce60d30486290b9a91f0735254e31329f02e1ee291050" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 28871, - "sha256": "3bf99aca0798e86e13a5364aaf8462f23cdb472770671f4bef41ac50abf65c32" + "bytes": 28935, + "sha256": "a7b7809d3b423bf647a528c3b0f6991f2ea3874afdd93a4d6ba74eb189131f2a" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 17521, - "sha256": "69b7ba66e5e2710438d168e2e9e9c5a6187c9f8dadc6a1e7a0f60e4dc222d679" + "bytes": 17576, + "sha256": "d72777b421b57f0cf8f31b4d048cf92512e49216d0b0c05cec1686d4aea638ce" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 23958, - "sha256": "46b435f98d8a91501580fa1426a8ab0d0f57b3bc92ecd17f42332e655fd2bd85" + "bytes": 24019, + "sha256": "73d8ef2898f25ae899a618b723f40a6907646c303effa21cbde6156d01a95fc7" }, { "label": "world_class_claim_guard", @@ -17955,9 +17977,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1121225, + "other_text_tokens": 1121243, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1122022, + "estimated_total_text_tokens": 1122040, "deferred_resource_tokens": 512782, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ @@ -19933,7 +19955,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16T20:58:58Z", + "generated_at": "2026-06-17", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -19957,14 +19979,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -19973,7 +19995,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -19985,31 +20007,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "skill_activation", + "event": "review_event", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } @@ -21409,8 +21431,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -21426,8 +21448,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -21879,7 +21901,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -21916,7 +21938,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -21937,8 +21959,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -21954,8 +21976,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -22323,7 +22345,7 @@ "source_accepted": false, "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -22407,14 +22429,14 @@ "collection_ready_count": 1, "collection_blocked_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, - "repair_checklist_count": 13, - "repair_blocked_count": 13, + "source_pass_count": 9, + "source_blocked_count": 10, + "repair_checklist_count": 14, + "repair_blocked_count": 14, "repair_ready_count": 0, "repair_phase_counts": { "unblock-access": 4, - "collect-source": 9 + "collect-source": 10 }, "next_repair_action_id": "human-adjudication-precheck-human-reviewer", "next_repair_phase": "unblock-access", @@ -22423,7 +22445,7 @@ "repair_counts_as_completion": false, "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 13, + "phase_queue_row_count": 14, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -23469,8 +23491,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -23554,6 +23576,21 @@ "source": "prechecks", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -23611,19 +23648,34 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", - "next_action": "Import at least one metadata-only event from a real client.", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -23989,8 +24041,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -24141,6 +24193,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -24294,8 +24361,8 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 9, - "row_count": 9, + "blocked_count": 10, + "row_count": 10, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", @@ -24388,6 +24455,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -24519,8 +24601,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "review_counts_submission_as_completion": false, "summary": { "review_item_count": 4, @@ -24532,8 +24614,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "awaiting-submissions" @@ -24883,7 +24965,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -24904,8 +24986,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -24921,8 +25003,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", @@ -24959,8 +25041,8 @@ "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "runbook_counts_as_completion": false, "decision": "collect-evidence" @@ -25434,7 +25516,7 @@ "review_state": "awaiting-submission", "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "execution_runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -25482,7 +25564,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -25503,8 +25585,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -25519,9 +25601,10 @@ "next_action": "Telemetry must stay metadata-only." } ], - "blocked_source_check_count": 1, + "blocked_source_check_count": 2, "next_source_actions": [ - "Import at least one metadata-only event from a real client." + "Import at least one metadata-only event from a real client.", + "Telemetry must include adoption outcome evidence." ], "submission_state": { "status": "missing", diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 4b301d4..6b2a8d7 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -412,7 +412,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -472,7 +472,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -1009,7 +1010,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf", + "evidence_bundle_sha256": "2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403", "source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -1026,18 +1027,18 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4, "working_tree_dirty": true, - "changed_file_count": 17, + "changed_file_count": 18, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 17 + "generated_changed_file_count": 18 }, - "commit": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "commit": "28c48ec248807020a68b50e012d4419edafbb418", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", @@ -1329,14 +1330,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1345,7 +1346,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1620,7 +1621,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1673,8 +1674,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -2100,7 +2101,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2137,7 +2138,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2158,8 +2159,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2175,8 +2176,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/skill-interpretation.html b/reports/skill-interpretation.html index 558d3e1..64da657 100644 --- a/reports/skill-interpretation.html +++ b/reports/skill-interpretation.html @@ -929,7 +929,7 @@ -

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks10 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks9 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index 58188dd..368c318 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -412,7 +412,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -472,7 +472,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -1013,7 +1014,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf", + "evidence_bundle_sha256": "2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403", "source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -1030,18 +1031,18 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4, "working_tree_dirty": true, - "changed_file_count": 17, + "changed_file_count": 18, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 17 + "generated_changed_file_count": 18 }, - "commit": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "commit": "28c48ec248807020a68b50e012d4419edafbb418", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", @@ -1333,14 +1334,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1349,7 +1350,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1624,7 +1625,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1677,8 +1678,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -2104,7 +2105,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2141,7 +2142,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2162,8 +2163,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2179,8 +2180,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/skill-overview.html b/reports/skill-overview.html index b93d300..8eab135 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -929,7 +929,7 @@ -

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks10 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks9 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index e49139c..4af453c 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -411,7 +411,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 19, - "source_pass_count": 10, + "source_pass_count": 9, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -471,7 +471,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -1008,7 +1009,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "98271c40046d1bf5f6a3ada701b5b1fd88b188bf4122a15424c8b4566cfc8fcf", + "evidence_bundle_sha256": "2c4ae0e9054c4680f9239be6ec01010ee3f4e72d858c20823a45f6abdd969403", "source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -1025,18 +1026,18 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 19, - "world_class_source_pass_count": 10, - "world_class_source_blocked_count": 9, + "world_class_source_pass_count": 9, + "world_class_source_blocked_count": 10, "public_claim_ready": false, "public_claim_blocker_count": 4, "working_tree_dirty": true, - "changed_file_count": 17, + "changed_file_count": 18, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 17 + "generated_changed_file_count": 18 }, - "commit": "9eaa8f6cec4ba83c07ded0275686252576bee6c3", + "commit": "28c48ec248807020a68b50e012d4419edafbb418", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", @@ -1328,14 +1329,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1344,7 +1345,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1619,7 +1620,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1672,8 +1673,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -2099,7 +2100,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2136,7 +2137,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2157,8 +2158,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2174,8 +2175,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index 09c5958..1bfa182 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -328,7 +328,7 @@ "source_accepted": false, "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index cab44e9..d411c67 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -16,8 +16,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -33,8 +33,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -486,7 +486,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -523,7 +523,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -544,8 +544,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -561,8 +561,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index 82aa5c4..8644cbf 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -8,8 +8,8 @@ Generated at: `2026-06-17` - ready to claim world-class: `false` - entries: `4` - source accepted: `0` -- source checks: `10` pass / `19` total -- source blocked: `9` +- source checks: `9` pass / `19` total +- source blocked: `10` - accepted: `0` - pending: `4` - human pending: `1` @@ -30,7 +30,7 @@ This ledger records the current evidence state. It requires both passing source | `provider-holdout` | `pending` | `missing` | `external` | model-executed 0; token-observed 0 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | `human-adjudication` | `pending` | `missing` | `human` | 0/5 decisions; pending 5 | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | | `native-permission-enforcement` | `pending` | `missing` | `external` | native-enforced targets 0; installer-enforced targets 4 | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | -| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 1 | Install a real client against the native host and import production metadata-only events. | +| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. | ## Provider Holdout @@ -186,8 +186,8 @@ This ledger records the current evidence state. It requires both passing source - objective: Import production metadata-only events from a real external client into the local drift loop. - source status: `external_required` -- observed state: `{"external_source_events": 0, "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` -- source checks: `2` pass / `3` total +- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` +- source checks: `1` pass / `3` total - submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "ledger_counts_as_completion": false}` ### Provenance Requirements @@ -211,7 +211,7 @@ This ledger records the current evidence state. It requires both passing source | Check | Current | Expected | Status | | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | -| Adoption sample | `1` | `>0` | `pass` | +| Adoption sample | `0` | `>0` | `blocked` | | Raw content blocked | `False` | `false` | `pass` | ### Completion Assertions diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index bbb6e41..1fc0097 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -154,7 +154,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -325,7 +325,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index 5b43964..c95cc73 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -22,7 +22,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It | `provider-holdout` | `external_required` | `external` | operator with provider credentials | model-executed 0; token-observed 0 | | `human-adjudication` | `human_required` | `human` | human reviewer | 0/5 decisions; pending 5 | | `native-permission-enforcement` | `external_required` | `external` | target client or installer integrator | native-enforced targets 0; installer-enforced targets 4 | -| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 1 | +| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 0 | ## Provider Holdout diff --git a/reports/world_class_evidence_preflight.html b/reports/world_class_evidence_preflight.html index 1cdcc21..4878b3a 100644 --- a/reports/world_class_evidence_preflight.html +++ b/reports/world_class_evidence_preflight.html @@ -68,7 +68,7 @@ Evidence Collection

World-Class Evidence Preflight

This operator view shows which external and human evidence is still blocked, which commands prepare editable submission drafts, and why preflight never counts as accepted evidence.

-
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source10/19
Repairs13/13
Phases2/2
+
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source9/19
Repairs14/14
Phases2/2
@@ -128,7 +128,7 @@
Phase
collect-source
-
Rows
9 / 9 blocked
+
Rows
10 / 10 blocked
Owners
Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator
Evidence
human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout
Next
Set reviewer_attestation only after choices are completed before opening the answer key.
@@ -928,15 +928,15 @@
-
+
adoption_sample_count Adoption sample
-
Current
1
+
Current
0
Expected
>0
-
Status
pass
+
Status
blocked
Action
Telemetry must include adoption outcome evidence.
@@ -975,6 +975,23 @@
+
+
+ source-check + adoption_sample_count +
+
+
Priority
40
+
Phase
collect-source
+
Owner
Browser/Chrome/IDE/provider client integrator
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Telemetry must include adoption outcome evidence.
+
Verify
python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+
source-check diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index 80f304f..57bd4d0 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -14,14 +14,14 @@ "collection_ready_count": 1, "collection_blocked_count": 3, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, - "repair_checklist_count": 13, - "repair_blocked_count": 13, + "source_pass_count": 9, + "source_blocked_count": 10, + "repair_checklist_count": 14, + "repair_blocked_count": 14, "repair_ready_count": 0, "repair_phase_counts": { "unblock-access": 4, - "collect-source": 9 + "collect-source": 10 }, "next_repair_action_id": "human-adjudication-precheck-human-reviewer", "next_repair_phase": "unblock-access", @@ -30,7 +30,7 @@ "repair_counts_as_completion": false, "phase_queue_count": 2, "phase_queue_blocked_count": 2, - "phase_queue_row_count": 13, + "phase_queue_row_count": 14, "phase_queue_next_phase": "unblock-access", "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", @@ -1076,8 +1076,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -1161,6 +1161,21 @@ "source": "prechecks", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1218,19 +1233,34 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 1, - "row_count": 1, + "blocked_count": 2, + "row_count": 2, "owners": [ "Browser/Chrome/IDE/provider client integrator" ], "evidence_keys": [ "native-client-telemetry" ], - "next_action_id": "native-client-telemetry-source-check-external_source_events", - "next_action": "Import at least one metadata-only event from a real client.", + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", "counts_as_completion": false, "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1596,8 +1626,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -1748,6 +1778,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", @@ -1901,8 +1946,8 @@ "label": "Collect source", "priority": 40, "status": "blocked", - "blocked_count": 9, - "row_count": 9, + "blocked_count": 10, + "row_count": 10, "owners": [ "Browser/Chrome/IDE/provider client integrator", "human reviewer", @@ -1995,6 +2040,21 @@ "source": "source_checklist", "counts_as_completion": false }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, { "action_id": "native-client-telemetry-source-check-external_source_events", "evidence_key": "native-client-telemetry", diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index d7343ba..f18b77b 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -10,10 +10,10 @@ Generated at: `2026-06-17` - credential value exposed: `false` - collection ready: `1` - collection blocked: `3` -- source checks: `10` pass / `19` total -- repair rows: `13` blocked / `13` total +- source checks: `9` pass / `19` total +- repair rows: `14` blocked / `14` total - phase queue: `2` blocked / `2` phases -- phase queue rows: `13` +- phase queue rows: `14` - next repair action: `human-adjudication-precheck-human-reviewer` - next repair owner: `human reviewer` - next phase: `unblock-access` @@ -51,7 +51,7 @@ Phase queue rows group the same repair checklist into operator execution phases. | Priority | Phase | Status | Rows | Owners | Evidence | Verify | Next action | | --- | --- | --- | --- | --- | --- | --- | --- | | `20` | `unblock-access` | `blocked` | 4 / 4 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Assign a real reviewer identity before claiming human adjudication. | -| `40` | `collect-source` | `blocked` | 9 / 9 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. | +| `40` | `collect-source` | `blocked` | 10 / 10 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. | ## Evidence Items @@ -77,6 +77,7 @@ Repair rows convert preflight and source blockers into a prioritized operator qu | `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `pending_count` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Record a reviewer choice and reason for every pair. | | `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `ready_for_human_evidence` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints. | | `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `reviewer_metadata_present` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Record reviewer and reviewed_at before adjudication can count. | +| `40` | `collect-source` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `source-check` | `adoption_sample_count` | `blocked` | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Telemetry must include adoption outcome evidence. | | `40` | `collect-source` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `source-check` | `external_source_events` | `blocked` | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Import at least one metadata-only event from a real client. | | `40` | `collect-source` | target client or installer integrator | `native-permission-enforcement` | `source-check` | `native_enforcement_count` | `blocked` | `python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Collect real target-client or external runtime guard proof. | | `40` | `collect-source` | operator with provider credentials | `provider-holdout` | `source-check` | `model_executed_count` | `blocked` | `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Run provider-backed output-exec with real credentials. | @@ -192,7 +193,7 @@ Repair rows convert preflight and source blockers into a prioritized operator qu | Check | Current | Expected | Status | Next action | | --- | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | -| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. | +| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | | Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | ## Boundary diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html index 8f7c914..05be43e 100644 --- a/reports/world_class_operator_runbook.html +++ b/reports/world_class_operator_runbook.html @@ -57,7 +57,7 @@ Evidence Operations

World-Class Operator Runbook

A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

-
Pending4
Awaiting4
Ready0
Source10/19
Blocked9
Invalid0
+
Pending4
Awaiting4
Ready0
Source9/19
Blocked10
Invalid0

Fast Path

  1. Run the real external or human work for one evidence item.
  2. Generate and fill the matching submission draft.
  3. Validate intake and inspect the submission review queue.
  4. Refresh the ledger and run the claim guard before making any completion claim.

Evidence Items

@@ -120,7 +120,7 @@
Owner
Browser/Chrome/IDE/provider client integrator
Ledger
pending
-
Blocked
1
+
Blocked
2
Submission
evidence/world_class/submissions/native-client-telemetry.json

Source Runbook

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
@@ -130,8 +130,8 @@

Success Checks

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

Privacy

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
-

Next Source Actions

  • Import at least one metadata-only event from a real client.
-

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 1 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+

Next Source Actions

  • Import at least one metadata-only event from a real client.
  • Telemetry must include adoption outcome evidence.
+

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.

Boundary

  • Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.
  • Valid intake means ready for submission review; ledger review still requires passing source evidence.
  • The world-class ledger and claim guard remain the source of truth.
diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index 24e2bd7..0c82d92 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -12,8 +12,8 @@ "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "runbook_counts_as_completion": false, "decision": "collect-evidence" @@ -487,7 +487,7 @@ "review_state": "awaiting-submission", "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "execution_runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -535,7 +535,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -556,8 +556,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -572,9 +572,10 @@ "next_action": "Telemetry must stay metadata-only." } ], - "blocked_source_check_count": 1, + "blocked_source_check_count": 2, "next_source_actions": [ - "Import at least one metadata-only event from a real client." + "Import at least one metadata-only event from a real client.", + "Telemetry must include adoption outcome evidence." ], "submission_state": { "status": "missing", diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index 833c4d2..25c0dde 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -28,7 +28,7 @@ This runbook coordinates evidence collection only. It does not accept submission | `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials. | operator with provider credentials | | `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | `5` | Record a reviewer choice and reason for every pair. | human reviewer | | `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Collect real target-client or external runtime guard proof. | target client or installer integrator | -| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | +| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | ## Provider Holdout @@ -261,7 +261,7 @@ This runbook coordinates evidence collection only. It does not accept submission - objective: Import production metadata-only events from a real external client into the local drift loop. - blocking reason: No evidence packet has been submitted for review. -- blocked source checks: `1` +- blocked source checks: `2` - submission: `evidence/world_class/submissions/native-client-telemetry.json` - template: `evidence/world_class/templates/native-client-telemetry.intake.json` @@ -314,13 +314,14 @@ This runbook coordinates evidence collection only. It does not accept submission ### Next Source Actions - Import at least one metadata-only event from a real client. +- Telemetry must include adoption outcome evidence. ### Source Evidence Snapshot | Check | Current | Expected | Status | Next action | | --- | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | -| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. | +| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | | Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | ## Boundary diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index 3c09e53..4c9a96e 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -14,8 +14,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "review_counts_submission_as_completion": false, "summary": { "review_item_count": 4, @@ -27,8 +27,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 19, - "source_pass_count": 10, - "source_blocked_count": 9, + "source_pass_count": 9, + "source_blocked_count": 10, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "awaiting-submissions" @@ -378,7 +378,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -399,8 +399,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -416,8 +416,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 2dda5c0..a6b7a5d 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -153,7 +153,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Source Checks - External events: 0 / >0 => blocked -- Adoption sample: 1 / >0 => pass +- Adoption sample: 0 / >0 => blocked - Raw content blocked: False / false => pass #### Completion Assertions