Mirror phase queues into operator runbook
This commit is contained in:
yaojingang
+1
-1
@@ -16,7 +16,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2"
|
||||
},
|
||||
"compatibility": {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-17",
|
||||
"generated_at": "2026-06-16T21:09:22Z",
|
||||
"skill_dir": ".",
|
||||
"source_patterns": "reports/user_patterns.json",
|
||||
"pattern_count": 5,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Adaptation Proposals
|
||||
|
||||
- Generated at: `2026-06-17`
|
||||
- Generated at: `2026-06-16T21:09:22Z`
|
||||
- Pattern report: `reports/user_patterns.json`
|
||||
- Proposal only: `true`
|
||||
- Writes repository files: `false`
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-17",
|
||||
"generated_at": "2026-06-16T21:09:51Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
@@ -25,14 +25,14 @@
|
||||
},
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -41,7 +41,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -53,31 +53,31 @@
|
||||
{
|
||||
"skill": "yao-meta-skill",
|
||||
"events": 1,
|
||||
"adoption_events": 0,
|
||||
"accepted": 0,
|
||||
"adoption_events": 1,
|
||||
"accepted": 1,
|
||||
"edited": 0,
|
||||
"rejected": 0,
|
||||
"missed": 0,
|
||||
"adoption_rate": 0
|
||||
"adoption_rate": 100.0
|
||||
}
|
||||
],
|
||||
"next_iteration_candidates": [],
|
||||
"recent_events": [
|
||||
{
|
||||
"command": "unknown",
|
||||
"event": "review_event",
|
||||
"event": "skill_activation",
|
||||
"skill": "yao-meta-skill",
|
||||
"source": "manual",
|
||||
"version": "1.1.0",
|
||||
"activation_type": "manual",
|
||||
"outcome": "reviewed",
|
||||
"activation_type": "explicit",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"timestamp": "2026-06-13T12:00:00Z"
|
||||
"timestamp": "2026-06-13T10:00:00Z"
|
||||
}
|
||||
],
|
||||
"failures": [],
|
||||
"artifacts": {
|
||||
"events_jsonl": "reports/telemetry_events.jsonl",
|
||||
"events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl",
|
||||
"json": "reports/adoption_drift_report.json",
|
||||
"markdown": "reports/adoption_drift_report.md"
|
||||
}
|
||||
|
||||
@@ -5,9 +5,9 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs,
|
||||
## Summary
|
||||
|
||||
- Events: `1`
|
||||
- Adoption samples: `0`
|
||||
- Activation events: `0`
|
||||
- Adoption rate: `0`
|
||||
- Adoption samples: `1`
|
||||
- Activation events: `1`
|
||||
- Adoption rate: `100.0`
|
||||
- Missed trigger signals: `0`
|
||||
- Bad output signals: `0`
|
||||
- Script error signals: `0`
|
||||
@@ -25,7 +25,7 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs,
|
||||
|
||||
| Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| `yao-meta-skill` | 1 | 0 | 0 | 0 | 0 | 0 | 0 |
|
||||
| `yao-meta-skill` | 1 | 1 | 1 | 0 | 0 | 0 | 100.0 |
|
||||
|
||||
## Next Iteration Candidates
|
||||
|
||||
@@ -33,4 +33,4 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs,
|
||||
|
||||
## Recent Metadata Events
|
||||
|
||||
- `2026-06-13T12:00:00Z` `yao-meta-skill` event=`review_event` source=`manual` command=`unknown` activation=`manual` outcome=`reviewed` failure=`none`
|
||||
- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` source=`manual` command=`unknown` activation=`explicit` outcome=`accepted` failure=`none`
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_evidence_consistency.py",
|
||||
"lines": 609,
|
||||
"lines": 625,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"early_watch": true,
|
||||
@@ -90,7 +90,7 @@
|
||||
},
|
||||
{
|
||||
"path": "scripts/render_evidence_consistency.py",
|
||||
"lines": 579,
|
||||
"lines": 580,
|
||||
"kind": "cli-script",
|
||||
"severity": "pass",
|
||||
"early_watch": false,
|
||||
@@ -141,7 +141,7 @@
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_evidence_consistency.py",
|
||||
"lines": 609,
|
||||
"lines": 625,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"early_watch": true,
|
||||
|
||||
@@ -37,7 +37,7 @@ No near-threshold files found.
|
||||
| --- | ---: | --- | --- |
|
||||
| `tests/verify_yao_cli.py` | `696` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `tests/verify_world_class_evidence_intake.py` | `690` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `tests/verify_evidence_consistency.py` | `609` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `tests/verify_evidence_consistency.py` | `625` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
|
||||
## Largest Files
|
||||
|
||||
@@ -45,13 +45,13 @@ No near-threshold files found.
|
||||
| --- | ---: | --- | --- |
|
||||
| `tests/verify_yao_cli.py` | `696` | `test` | `pass` |
|
||||
| `tests/verify_world_class_evidence_intake.py` | `690` | `test` | `pass` |
|
||||
| `tests/verify_evidence_consistency.py` | `609` | `test` | `pass` |
|
||||
| `tests/verify_evidence_consistency.py` | `625` | `test` | `pass` |
|
||||
| `tests/verify_output_review_adjudication.py` | `599` | `test` | `pass` |
|
||||
| `scripts/render_skill_overview.py` | `588` | `cli-script` | `pass` |
|
||||
| `scripts/build_skill_atlas.py` | `586` | `cli-script` | `pass` |
|
||||
| `scripts/optimize_description.py` | `585` | `cli-script` | `pass` |
|
||||
| `scripts/trust_check.py` | `582` | `cli-script` | `pass` |
|
||||
| `scripts/render_evidence_consistency.py` | `579` | `cli-script` | `pass` |
|
||||
| `scripts/render_evidence_consistency.py` | `580` | `cli-script` | `pass` |
|
||||
| `tests/verify_world_class_evidence_ledger.py` | `579` | `test` | `pass` |
|
||||
| `scripts/world_class_submission_kit_rendering.py` | `576` | `internal-module` | `pass` |
|
||||
| `scripts/render_review_studio.py` | `568` | `cli-script` | `pass` |
|
||||
|
||||
@@ -3,18 +3,50 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-17",
|
||||
"skill_dir": ".",
|
||||
"commit": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"commit": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"generated_dirty": false,
|
||||
"generated_changed_file_count": 0,
|
||||
"source_dirty": false,
|
||||
"source_changed_file_count": 0,
|
||||
"sample": [],
|
||||
"source_sample": [],
|
||||
"generated_sample": [],
|
||||
"dirty": true,
|
||||
"changed_file_count": 49,
|
||||
"generated_dirty": true,
|
||||
"generated_changed_file_count": 44,
|
||||
"source_dirty": true,
|
||||
"source_changed_file_count": 5,
|
||||
"sample": [
|
||||
" M registry/index.json",
|
||||
" M registry/packages/yao-meta-skill.json",
|
||||
" M reports/adaptation_proposals.json",
|
||||
" M reports/adaptation_proposals.md",
|
||||
" M reports/adoption_drift_report.json",
|
||||
" M reports/adoption_drift_report.md",
|
||||
" M reports/architecture_maintainability.json",
|
||||
" M reports/architecture_maintainability.md",
|
||||
" M reports/benchmark_reproducibility.json",
|
||||
" M reports/benchmark_reproducibility.md",
|
||||
" M reports/context_budget.json",
|
||||
" M reports/context_budget.md"
|
||||
],
|
||||
"source_sample": [
|
||||
" M scripts/evidence_consistency_phase_queue.py",
|
||||
" M scripts/render_evidence_consistency.py",
|
||||
" M scripts/render_world_class_operator_runbook.py",
|
||||
" M tests/verify_evidence_consistency.py",
|
||||
" M tests/verify_world_class_operator_runbook.py"
|
||||
],
|
||||
"generated_sample": [
|
||||
" M registry/index.json",
|
||||
" M registry/packages/yao-meta-skill.json",
|
||||
" M reports/adaptation_proposals.json",
|
||||
" M reports/adaptation_proposals.md",
|
||||
" M reports/adoption_drift_report.json",
|
||||
" M reports/adoption_drift_report.md",
|
||||
" M reports/architecture_maintainability.json",
|
||||
" M reports/architecture_maintainability.md",
|
||||
" M reports/benchmark_reproducibility.json",
|
||||
" M reports/benchmark_reproducibility.md",
|
||||
" M reports/context_budget.json",
|
||||
" M reports/context_budget.md"
|
||||
],
|
||||
"generated_dirty_prefixes": [
|
||||
"dist/",
|
||||
"registry/index.json",
|
||||
@@ -27,12 +59,12 @@
|
||||
},
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "9080619d1499c827ab337b9172cc889106cf64f8c5a8baa86badcc8fc42ecf1a",
|
||||
"source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"evidence_bundle_sha256": "48071ebca4a8bfc9b45e5d4194858870fd0e7f50faff88d4abb79308113126eb",
|
||||
"source_contract_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
@@ -48,35 +80,36 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 19,
|
||||
"world_class_source_pass_count": 9,
|
||||
"world_class_source_blocked_count": 10,
|
||||
"world_class_source_pass_count": 10,
|
||||
"world_class_source_blocked_count": 9,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"source_tree_dirty": false,
|
||||
"source_changed_file_count": 0,
|
||||
"generated_tree_dirty": false,
|
||||
"generated_changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 49,
|
||||
"source_tree_dirty": true,
|
||||
"source_changed_file_count": 5,
|
||||
"generated_tree_dirty": true,
|
||||
"generated_changed_file_count": 44
|
||||
},
|
||||
"public_claim": {
|
||||
"ready": false,
|
||||
"scope": "public benchmark or world-class readiness claim",
|
||||
"blockers": [
|
||||
"release lock is not clean or commit is unavailable",
|
||||
"provider-backed model holdout evidence is incomplete",
|
||||
"human blind-review adjudication is incomplete",
|
||||
"world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)",
|
||||
"world-class source checks are not all accepted (9/19 pass, 10 blocked)"
|
||||
"world-class source checks are not all accepted (10/19 pass, 9 blocked)"
|
||||
],
|
||||
"policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks."
|
||||
},
|
||||
"release_lock": {
|
||||
"ready": true,
|
||||
"commit": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"ready": false,
|
||||
"commit": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"status_scope": "generation-time status before this report is written",
|
||||
"source_changed_file_count": 0,
|
||||
"generated_changed_file_count": 0,
|
||||
"reason": "clean source tree at generation-time HEAD"
|
||||
"source_changed_file_count": 5,
|
||||
"generated_changed_file_count": 44,
|
||||
"reason": "source files were dirty at generation time"
|
||||
},
|
||||
"evidence_bundle": {
|
||||
"algorithm": "sha256(path,label,exists,artifact_sha256)",
|
||||
@@ -84,7 +117,7 @@
|
||||
"existing_count": 25,
|
||||
"missing_count": 0,
|
||||
"missing_paths": [],
|
||||
"sha256": "9080619d1499c827ab337b9172cc889106cf64f8c5a8baa86badcc8fc42ecf1a"
|
||||
"sha256": "48071ebca4a8bfc9b45e5d4194858870fd0e7f50faff88d4abb79308113126eb"
|
||||
},
|
||||
"methodology": {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
@@ -157,8 +190,8 @@
|
||||
"label": "output_execution",
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"exists": true,
|
||||
"bytes": 7967,
|
||||
"sha256": "26c5abca0e771fbdbb1e27dd4fe4868d8b379a92df3d40517932ed5346c5100c"
|
||||
"bytes": 7966,
|
||||
"sha256": "7795806f55493c49185de218e3a40ef065be44aee0674ffe397c3debf41a278d"
|
||||
},
|
||||
{
|
||||
"label": "blind_review",
|
||||
@@ -193,7 +226,7 @@
|
||||
"path": "reports/security_trust_report.json",
|
||||
"exists": true,
|
||||
"bytes": 136999,
|
||||
"sha256": "a44deb1fb680973315225900381bb1d09b71352fca1bed3a99df60393340c6fc"
|
||||
"sha256": "e3bee7ca1d834b40432038b803311412613762086f35baa82a7fcf8545837213"
|
||||
},
|
||||
{
|
||||
"label": "python_compatibility",
|
||||
@@ -207,7 +240,7 @@
|
||||
"path": "reports/registry_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 3183,
|
||||
"sha256": "af63529bb20c444d93b81a278be60ff89b2f49245f0c557cf574cc2dfd6d1e2e"
|
||||
"sha256": "b4ecfad8408c2b1a00fe855cdebd71678bb5efbd5493d06fd01a64f6c4eb0800"
|
||||
},
|
||||
{
|
||||
"label": "package_verification",
|
||||
@@ -235,56 +268,56 @@
|
||||
"path": "reports/world_class_evidence_plan.json",
|
||||
"exists": true,
|
||||
"bytes": 22784,
|
||||
"sha256": "2beb888d7992fc72073251632a6e85c4075278f9bcea946a46a7e11f8f43e0ca"
|
||||
"sha256": "d0ab417742950ff5a5f3407f068829217da5c997a6a7c5a5ee59441cb4abfb7f"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_ledger",
|
||||
"path": "reports/world_class_evidence_ledger.json",
|
||||
"exists": true,
|
||||
"bytes": 26019,
|
||||
"sha256": "22c85613d7bef62c4d0a4c94b77a8596090dc55c40e1663370d16d903e0b1ffb"
|
||||
"bytes": 26016,
|
||||
"sha256": "fc4c3e398ad0de110ae5f37e461d03712716552ce85b44e41e09f4d343a17f0b"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_intake",
|
||||
"path": "reports/world_class_evidence_intake.json",
|
||||
"exists": true,
|
||||
"bytes": 20646,
|
||||
"sha256": "d3f44570e605e6e1b1557889ccca18efbbd8acc1d9a9c638f59961b3dc3e9026"
|
||||
"sha256": "1a939bdd1f4a3ee30e05f0b286145f0f3320ba9f1935749806a581737155774a"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_preflight",
|
||||
"path": "reports/world_class_evidence_preflight.json",
|
||||
"exists": true,
|
||||
"bytes": 100952,
|
||||
"sha256": "7bbbf51cfbaffbe606566c1eebfd8b3927e75b33efeea5132e1bd03382c897f9"
|
||||
"bytes": 97462,
|
||||
"sha256": "d6c13c020987c1b395d8e78cc409fec0784f6746ff77f0ef148abf105b9d6045"
|
||||
},
|
||||
{
|
||||
"label": "world_class_submission_review",
|
||||
"path": "reports/world_class_submission_review.json",
|
||||
"exists": true,
|
||||
"bytes": 17302,
|
||||
"sha256": "96624a5b7f79aa0ac43ce60d30486290b9a91f0735254e31329f02e1ee291050"
|
||||
"bytes": 17299,
|
||||
"sha256": "b256c40e75283556fc2ab1a0ff3a19834e82d54d4c7a3e7464806c01aab69db7"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook",
|
||||
"path": "reports/world_class_operator_runbook.json",
|
||||
"exists": true,
|
||||
"bytes": 28935,
|
||||
"sha256": "a7b7809d3b423bf647a528c3b0f6991f2ea3874afdd93a4d6ba74eb189131f2a"
|
||||
"bytes": 80422,
|
||||
"sha256": "4b5cd9ae73329dcf3787234d5d07fdcc773527849c9742206c52eabc2b081d24"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook_markdown",
|
||||
"path": "reports/world_class_operator_runbook.md",
|
||||
"exists": true,
|
||||
"bytes": 17576,
|
||||
"sha256": "d72777b421b57f0cf8f31b4d048cf92512e49216d0b0c05cec1686d4aea638ce"
|
||||
"bytes": 20156,
|
||||
"sha256": "5a95e78c737ebb2be0b13f45c5afc7e08035d3cc60b7adc49968845fcb5f6568"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook_html",
|
||||
"path": "reports/world_class_operator_runbook.html",
|
||||
"exists": true,
|
||||
"bytes": 24019,
|
||||
"sha256": "73d8ef2898f25ae899a618b723f40a6907646c303effa21cbde6156d01a95fc7"
|
||||
"bytes": 26522,
|
||||
"sha256": "d655a6d97fce416668b948894f1408b2f1bfb76068b0d738ab6d3fdd7112fff8"
|
||||
},
|
||||
{
|
||||
"label": "world_class_claim_guard",
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
# Benchmark Reproducibility
|
||||
|
||||
Generated at: `2026-06-17`
|
||||
Commit: `136654cda911836efd177294a17efc4e5bf08ae9`
|
||||
Working tree dirty at generation: `false`
|
||||
Source tree dirty at generation: `false`
|
||||
Generated evidence dirty at generation: `false`
|
||||
Evidence bundle SHA256: `9080619d1499c827ab337b9172cc889106cf64f8c5a8baa86badcc8fc42ecf1a`
|
||||
Commit: `6ece6407e81e352af8dffbb0933d8ee5128a92c3`
|
||||
Working tree dirty at generation: `true`
|
||||
Source tree dirty at generation: `true`
|
||||
Generated evidence dirty at generation: `true`
|
||||
Evidence bundle SHA256: `48071ebca4a8bfc9b45e5d4194858870fd0e7f50faff88d4abb79308113126eb`
|
||||
|
||||
## Summary
|
||||
|
||||
- reproducibility ready: `true`
|
||||
- release lock ready: `true`
|
||||
- release lock ready: `false`
|
||||
- methodology complete: `true`
|
||||
- required artifacts: `25`
|
||||
- missing artifacts: `0`
|
||||
- source contract sha256: `5533575c144f`
|
||||
- source contract sha256: `f57c75f37b2b`
|
||||
- archive sha256: `7db44f059844`
|
||||
- output cases: `5`
|
||||
- disclosed failure cases: `3`
|
||||
@@ -22,12 +22,12 @@ Evidence bundle SHA256: `9080619d1499c827ab337b9172cc889106cf64f8c5a8baa86badcc8
|
||||
- provider evidence complete: `false`
|
||||
- human review complete: `false`
|
||||
- world-class ready: `false`
|
||||
- world-class source checks: `9` pass / `19` total; `10` blocked
|
||||
- world-class source checks: `10` pass / `19` total; `9` blocked
|
||||
- public claim ready: `false`
|
||||
- public claim blockers: `4`
|
||||
- changed files at generation: `0`
|
||||
- source changed files at generation: `0`
|
||||
- generated changed files at generation: `0`
|
||||
- public claim blockers: `5`
|
||||
- changed files at generation: `49`
|
||||
- source changed files at generation: `5`
|
||||
- generated changed files at generation: `44`
|
||||
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.
|
||||
|
||||
@@ -39,22 +39,23 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
|
||||
| Blocker |
|
||||
| --- |
|
||||
| release lock is not clean or commit is unavailable |
|
||||
| provider-backed model holdout evidence is incomplete |
|
||||
| human blind-review adjudication is incomplete |
|
||||
| world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) |
|
||||
| world-class source checks are not all accepted (9/19 pass, 10 blocked) |
|
||||
| world-class source checks are not all accepted (10/19 pass, 9 blocked) |
|
||||
|
||||
## Release Lock
|
||||
|
||||
- ready: `true`
|
||||
- reason: clean source tree at generation-time HEAD
|
||||
- ready: `false`
|
||||
- reason: source files were dirty at generation time
|
||||
- status scope: generation-time status before this report is written
|
||||
|
||||
## Evidence Bundle
|
||||
|
||||
- algorithm: `sha256(path,label,exists,artifact_sha256)`
|
||||
- artifacts: `25` / `25`
|
||||
- sha256: `9080619d1499c827ab337b9172cc889106cf64f8c5a8baa86badcc8fc42ecf1a`
|
||||
- sha256: `48071ebca4a8bfc9b45e5d4194858870fd0e7f50faff88d4abb79308113126eb`
|
||||
|
||||
## Methodology Sections
|
||||
|
||||
@@ -76,25 +77,25 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
|
||||
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
|
||||
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `26c5abca0e77` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `7795806f5549` |
|
||||
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
|
||||
| review_adjudication | `reports/output_review_adjudication.json` | present | `91fd88dd9b0f` |
|
||||
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
|
||||
| runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `a44deb1fb680` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `e3bee7ca1d83` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `cb95afb7d7cc` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `af63529bb20c` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `b4ecfad8408c` |
|
||||
| package_verification | `reports/package_verification.json` | present | `7aa6c25e8154` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `d2bea19ea46f` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `ad57230481ed` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `2beb888d7992` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `22c85613d7be` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `d3f44570e605` |
|
||||
| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `7bbbf51cfbaf` |
|
||||
| world_class_submission_review | `reports/world_class_submission_review.json` | present | `96624a5b7f79` |
|
||||
| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `a7b7809d3b42` |
|
||||
| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `d72777b421b5` |
|
||||
| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `73d8ef2898f2` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `d0ab41774295` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `fc4c3e398ad0` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `1a939bdd1f4a` |
|
||||
| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `d6c13c020987` |
|
||||
| world_class_submission_review | `reports/world_class_submission_review.json` | present | `b256c40e7528` |
|
||||
| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `4b5cd9ae7332` |
|
||||
| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `5a95e78c737e` |
|
||||
| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `d655a6d97fce` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `abe7f7d60c00` |
|
||||
|
||||
## Reproduction Commands
|
||||
|
||||
@@ -6,15 +6,15 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 797,
|
||||
"other_text_tokens": 1121243,
|
||||
"other_text_tokens": 1137383,
|
||||
"estimated_initial_load_tokens": 990,
|
||||
"estimated_total_text_tokens": 1122040,
|
||||
"deferred_resource_tokens": 512782,
|
||||
"estimated_total_text_tokens": 1138180,
|
||||
"deferred_resource_tokens": 514654,
|
||||
"deferred_resource_warn_threshold": 120000,
|
||||
"deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 451479,
|
||||
"estimated_tokens": 453351,
|
||||
"file_count": 152
|
||||
},
|
||||
{
|
||||
@@ -36,7 +36,7 @@
|
||||
"large_deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 451479,
|
||||
"estimated_tokens": 453351,
|
||||
"file_count": 152
|
||||
}
|
||||
],
|
||||
@@ -59,7 +59,7 @@
|
||||
],
|
||||
"missing": [],
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 451479,
|
||||
"estimated_tokens": 453351,
|
||||
"file_count": 152,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
|
||||
| root | `.` | `production` | 1000 | 990 | 797 | 512782 | `governed` | scripts:451479 | 131.3 | - | ok |
|
||||
| root | `.` | `production` | 1000 | 990 | 797 | 514654 | `governed` | scripts:453351 | 131.3 | - | ok |
|
||||
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
|
||||
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"generated_at": "2026-06-17",
|
||||
"generated_at": "2026-03-31",
|
||||
"targets": [
|
||||
{
|
||||
"label": "root",
|
||||
@@ -8,11 +8,11 @@
|
||||
"budget_limit": 1000,
|
||||
"initial_tokens": 990,
|
||||
"skill_body_tokens": 797,
|
||||
"deferred_resource_tokens": 512782,
|
||||
"deferred_resource_tokens": 514654,
|
||||
"large_deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 451479,
|
||||
"estimated_tokens": 453351,
|
||||
"file_count": 152
|
||||
}
|
||||
],
|
||||
@@ -35,7 +35,7 @@
|
||||
],
|
||||
"missing": [],
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 451479,
|
||||
"estimated_tokens": 453351,
|
||||
"file_count": 152,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
|
||||
}
|
||||
|
||||
@@ -189,12 +189,12 @@
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"status": "pass",
|
||||
"detail": "initial load 990/1000; deferred 512782/120000; top deferred scripts 451479; resource governance governed; quality density 131.3",
|
||||
"detail": "initial load 990/1000; deferred 514654/120000; top deferred scripts 453351; resource governance governed; quality density 131.3",
|
||||
"evidence": "reports/context_budget.json"
|
||||
},
|
||||
"actual": {
|
||||
"status": "pass",
|
||||
"detail": "initial load 990/1000; deferred 512782/120000; top deferred scripts 451479; resource governance governed; quality density 131.3",
|
||||
"detail": "initial load 990/1000; deferred 514654/120000; top deferred scripts 453351; resource governance governed; quality density 131.3",
|
||||
"evidence": "reports/context_budget.json"
|
||||
},
|
||||
"paths": [
|
||||
@@ -207,8 +207,8 @@
|
||||
"key": "benchmark-release-lock-self-consistency",
|
||||
"label": "Benchmark release lock matches source dirty state",
|
||||
"status": "pass",
|
||||
"expected": true,
|
||||
"actual": true,
|
||||
"expected": false,
|
||||
"actual": false,
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json"
|
||||
],
|
||||
@@ -222,7 +222,7 @@
|
||||
"actual": {
|
||||
"available": true,
|
||||
"clean": false,
|
||||
"changed_file_count": 6
|
||||
"changed_file_count": 41
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json"
|
||||
@@ -263,8 +263,8 @@
|
||||
"key": "overview-benchmark-commit",
|
||||
"label": "overview embeds the benchmark commit",
|
||||
"status": "pass",
|
||||
"expected": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"actual": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"expected": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"actual": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-overview.json"
|
||||
@@ -276,30 +276,30 @@
|
||||
"label": "overview embeds benchmark summary fields",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"source_contract_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 19,
|
||||
"world_class_source_pass_count": 9,
|
||||
"world_class_source_blocked_count": 10,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"actual": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"source_contract_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 19,
|
||||
"world_class_source_pass_count": 9,
|
||||
"world_class_source_blocked_count": 10,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
@@ -409,8 +409,8 @@
|
||||
"key": "interpretation-benchmark-commit",
|
||||
"label": "interpretation embeds the benchmark commit",
|
||||
"status": "pass",
|
||||
"expected": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"actual": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"expected": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"actual": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-interpretation.json"
|
||||
@@ -422,30 +422,30 @@
|
||||
"label": "interpretation embeds benchmark summary fields",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"source_contract_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 19,
|
||||
"world_class_source_pass_count": 9,
|
||||
"world_class_source_blocked_count": 10,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"actual": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"source_contract_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2",
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 19,
|
||||
"world_class_source_pass_count": 9,
|
||||
"world_class_source_blocked_count": 10,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
@@ -2230,6 +2230,199 @@
|
||||
]
|
||||
},
|
||||
"phase_queue_counts_as_completion": false,
|
||||
"operator_runbook_summary": {
|
||||
"phase_queue_count": 2,
|
||||
"phase_queue_blocked_count": 2,
|
||||
"phase_queue_row_count": 14,
|
||||
"phase_queue_next_phase": "unblock-access",
|
||||
"phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer",
|
||||
"phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"phase_queue_counts_as_completion": false
|
||||
},
|
||||
"operator_runbook_top_level_phase_queue": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 4,
|
||||
"row_count": 4,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator",
|
||||
"human reviewer",
|
||||
"operator with provider credentials",
|
||||
"target client or installer integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"next_action_id": "human-adjudication-precheck-human-reviewer",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 10,
|
||||
"row_count": 10,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator",
|
||||
"human reviewer",
|
||||
"operator with provider credentials",
|
||||
"target client or installer integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"next_action_id": "human-adjudication-source-check-blind_review_attested",
|
||||
"verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"operator_runbook_phase_queues": {
|
||||
"provider-holdout": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"operator with provider credentials"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"provider-holdout"
|
||||
],
|
||||
"next_action_id": "provider-holdout-precheck-openai-api-key",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 2,
|
||||
"row_count": 2,
|
||||
"owners": [
|
||||
"operator with provider credentials"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"provider-holdout"
|
||||
],
|
||||
"next_action_id": "provider-holdout-source-check-model_executed_count",
|
||||
"verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"human-adjudication": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"human reviewer"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"human-adjudication"
|
||||
],
|
||||
"next_action_id": "human-adjudication-precheck-human-reviewer",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 5,
|
||||
"row_count": 5,
|
||||
"owners": [
|
||||
"human reviewer"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"human-adjudication"
|
||||
],
|
||||
"next_action_id": "human-adjudication-source-check-blind_review_attested",
|
||||
"verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"native-permission-enforcement": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"target client or installer integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-permission-enforcement"
|
||||
],
|
||||
"next_action_id": "native-permission-enforcement-precheck-native-guard",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"target client or installer integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-permission-enforcement"
|
||||
],
|
||||
"next_action_id": "native-permission-enforcement-source-check-native_enforcement_count",
|
||||
"verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"native-client-telemetry": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-client-telemetry"
|
||||
],
|
||||
"next_action_id": "native-client-telemetry-precheck-external-client",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 2,
|
||||
"row_count": 2,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-client-telemetry"
|
||||
],
|
||||
"next_action_id": "native-client-telemetry-source-check-adoption_sample_count",
|
||||
"verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"review_studio_phase_queues": {
|
||||
"provider-holdout": [
|
||||
{
|
||||
@@ -2425,6 +2618,61 @@
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"operator_runbook_summary": {
|
||||
"phase_queue_count": 2,
|
||||
"phase_queue_blocked_count": 2,
|
||||
"phase_queue_row_count": 14,
|
||||
"phase_queue_next_phase": "unblock-access",
|
||||
"phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer",
|
||||
"phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"phase_queue_counts_as_completion": false
|
||||
},
|
||||
"operator_runbook_top_level_phase_queue": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 4,
|
||||
"row_count": 4,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator",
|
||||
"human reviewer",
|
||||
"operator with provider credentials",
|
||||
"target client or installer integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"next_action_id": "human-adjudication-precheck-human-reviewer",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 10,
|
||||
"row_count": 10,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator",
|
||||
"human reviewer",
|
||||
"operator with provider credentials",
|
||||
"target client or installer integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"human-adjudication",
|
||||
"native-client-telemetry",
|
||||
"native-permission-enforcement",
|
||||
"provider-holdout"
|
||||
],
|
||||
"next_action_id": "human-adjudication-source-check-blind_review_attested",
|
||||
"verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"item_phase_queues": {
|
||||
"provider-holdout": [
|
||||
{
|
||||
@@ -2563,6 +2811,144 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"operator_runbook_phase_queues": {
|
||||
"provider-holdout": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"operator with provider credentials"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"provider-holdout"
|
||||
],
|
||||
"next_action_id": "provider-holdout-precheck-openai-api-key",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 2,
|
||||
"row_count": 2,
|
||||
"owners": [
|
||||
"operator with provider credentials"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"provider-holdout"
|
||||
],
|
||||
"next_action_id": "provider-holdout-source-check-model_executed_count",
|
||||
"verification_command": "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"human-adjudication": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"human reviewer"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"human-adjudication"
|
||||
],
|
||||
"next_action_id": "human-adjudication-precheck-human-reviewer",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 5,
|
||||
"row_count": 5,
|
||||
"owners": [
|
||||
"human reviewer"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"human-adjudication"
|
||||
],
|
||||
"next_action_id": "human-adjudication-source-check-blind_review_attested",
|
||||
"verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"native-permission-enforcement": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"target client or installer integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-permission-enforcement"
|
||||
],
|
||||
"next_action_id": "native-permission-enforcement-precheck-native-guard",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"target client or installer integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-permission-enforcement"
|
||||
],
|
||||
"next_action_id": "native-permission-enforcement-source-check-native_enforcement_count",
|
||||
"verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
],
|
||||
"native-client-telemetry": [
|
||||
{
|
||||
"phase": "unblock-access",
|
||||
"priority": 20,
|
||||
"status": "blocked",
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-client-telemetry"
|
||||
],
|
||||
"next_action_id": "native-client-telemetry-precheck-external-client",
|
||||
"verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 2,
|
||||
"row_count": 2,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-client-telemetry"
|
||||
],
|
||||
"next_action_id": "native-client-telemetry-source-check-adoption_sample_count",
|
||||
"verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"review_studio_phase_queues": {
|
||||
"provider-holdout": [
|
||||
{
|
||||
@@ -2705,9 +3091,10 @@
|
||||
},
|
||||
"paths": [
|
||||
"reports/world_class_evidence_preflight.json",
|
||||
"reports/world_class_operator_runbook.json",
|
||||
"reports/review-studio.json"
|
||||
],
|
||||
"detail": "Phase queues must be derived from repair rows in preflight and mirrored into Review Studio without counting queue guidance as completion evidence."
|
||||
"detail": "Phase queues must be derived from repair rows in preflight and mirrored into the operator runbook and Review Studio without counting queue guidance as completion evidence."
|
||||
},
|
||||
{
|
||||
"key": "review-studio-no-overclaim",
|
||||
|
||||
@@ -48,7 +48,7 @@ This gate compares generated evidence reports against each other. It does not cr
|
||||
| Preflight mirrors ledger without accepting evidence | `pass` | Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness. | `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_preflight.json` |
|
||||
| Preflight exposes a safe submission-kit handoff | `pass` | Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence. | `reports/world_class_evidence_preflight.json`, `reports/world_class_evidence_preflight.html` |
|
||||
| Review Studio mirrors preflight artifact roles | `pass` | The Review Studio world-class action card must carry the same submission-ref versus supporting-evidence contract as the preflight handoff. | `reports/world_class_evidence_preflight.json`, `reports/review-studio.json` |
|
||||
| World-class phase queues mirror repair rows | `pass` | Phase queues must be derived from repair rows in preflight and mirrored into Review Studio without counting queue guidance as completion evidence. | `reports/world_class_evidence_preflight.json`, `reports/review-studio.json` |
|
||||
| World-class phase queues mirror repair rows | `pass` | Phase queues must be derived from repair rows in preflight and mirrored into the operator runbook and Review Studio without counting queue guidance as completion evidence. | `reports/world_class_evidence_preflight.json`, `reports/world_class_operator_runbook.json`, `reports/review-studio.json` |
|
||||
| Review Studio does not overclaim pending world-class evidence | `pass` | When world-class evidence is pending, Review Studio must stay in a review or warning posture. | `reports/world_class_evidence_ledger.json`, `reports/review-studio.json` |
|
||||
| Claim guard covers package and runtime claim surfaces | `pass` | The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted. | `reports/world_class_claim_guard.json`, `manifest.json`, `agents/interface.yaml`, `dist/manifest.json`, `dist/targets/openai/adapter.json`, `evidence/world_class/README.md`, `security/permission_policy.json`, `reports/world_class_evidence_ledger.json` |
|
||||
| World-class evidence workflows cover every pending ledger entry | `pass` | Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion. | `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_plan.json`, `reports/world_class_evidence_intake.json`, `reports/world_class_submission_review.json`, `reports/world_class_operator_runbook.json`, `reports/review-studio.json` |
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.66,
|
||||
"duration_ms": 31.59,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -62,7 +62,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.23,
|
||||
"duration_ms": 28.83,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -85,7 +85,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.95,
|
||||
"duration_ms": 28.22,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -113,7 +113,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.04,
|
||||
"duration_ms": 28.41,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -136,7 +136,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.48,
|
||||
"duration_ms": 28.5,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -164,7 +164,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.24,
|
||||
"duration_ms": 28.37,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -187,7 +187,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.96,
|
||||
"duration_ms": 29.23,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -214,7 +214,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.05,
|
||||
"duration_ms": 28.25,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -237,7 +237,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.84,
|
||||
"duration_ms": 27.97,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -266,7 +266,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.86,
|
||||
"duration_ms": 28.02,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
|
||||
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
|
||||
|
||||
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
|
||||
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 26.66 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 26.23 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 25.95 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.04 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 28.48 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.24 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.96 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.05 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.84 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.86 | 98 | 100.0 | pass |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 31.59 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 28.83 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 28.22 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.41 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 28.5 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.37 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 29.23 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.25 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 27.97 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.02 | 98 | 100.0 | pass |
|
||||
|
||||
## Next Fixes
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -78,7 +78,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
- Maturity: `governed`
|
||||
- Owner: `Yao Team`
|
||||
- License: `MIT`
|
||||
- Package SHA256: `5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d`
|
||||
- Package SHA256: `f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f`
|
||||
- Archive SHA256: `7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2`
|
||||
- Install simulated: `True`
|
||||
|
||||
|
||||
+13
-13
File diff suppressed because one or more lines are too long
+1248
-260
File diff suppressed because it is too large
Load Diff
+32
-33
@@ -412,7 +412,7 @@
|
||||
"external_pending_count": 3,
|
||||
"human_pending_count": 1,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_pass_count": 10,
|
||||
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
|
||||
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
|
||||
"entries": [
|
||||
@@ -472,8 +472,7 @@
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
"blocked_checks": [
|
||||
"External events",
|
||||
"Adoption sample"
|
||||
"External events"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1006,12 +1005,12 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "9080619d1499c827ab337b9172cc889106cf64f8c5a8baa86badcc8fc42ecf1a",
|
||||
"source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"evidence_bundle_sha256": "48071ebca4a8bfc9b45e5d4194858870fd0e7f50faff88d4abb79308113126eb",
|
||||
"source_contract_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
@@ -1027,18 +1026,18 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 19,
|
||||
"world_class_source_pass_count": 9,
|
||||
"world_class_source_blocked_count": 10,
|
||||
"world_class_source_pass_count": 10,
|
||||
"world_class_source_blocked_count": 9,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"source_tree_dirty": false,
|
||||
"source_changed_file_count": 0,
|
||||
"generated_tree_dirty": false,
|
||||
"generated_changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 49,
|
||||
"source_tree_dirty": true,
|
||||
"source_changed_file_count": 5,
|
||||
"generated_tree_dirty": true,
|
||||
"generated_changed_file_count": 44
|
||||
},
|
||||
"commit": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"commit": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
|
||||
@@ -1143,7 +1142,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 244,
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1181,7 +1180,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -1313,7 +1312,7 @@
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"to": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1330,14 +1329,14 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -1346,7 +1345,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -1621,7 +1620,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1674,8 +1673,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_blocked_count": 10,
|
||||
"source_pass_count": 10,
|
||||
"source_blocked_count": 9,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -2101,7 +2100,7 @@
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -2138,7 +2137,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -2159,8 +2158,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -2176,8 +2175,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
"path": "evidence/world_class/submissions/native-client-telemetry.json",
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 244,
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
- Interactive scripts: `0`
|
||||
- Package hash scope: `source-contract-without-generated-reports`
|
||||
- Package hash files: `244`
|
||||
- Package SHA256: `5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d`
|
||||
- Package SHA256: `f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f`
|
||||
|
||||
## Failures
|
||||
|
||||
|
||||
@@ -929,7 +929,7 @@
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<article class='panel world-readiness'><div class='world-readiness-head'><div><h3><span data-lang="zh-CN">世界证据</span><span data-lang="en">World Evidence</span></h3><p><span data-lang="zh-CN">世界级证据尚未完成：4 项待补，0 项已接受。</span><span data-lang="en">World-class evidence is not complete: 4 pending, 0 accepted.</span></p></div><span class='world-status'><span data-lang="zh-CN">证据待补</span><span data-lang="en">Evidence pending</span></span></div><div class='evidence-kpis'><article class='evidence-kpi'><span><span data-lang="zh-CN">待补证据</span><span data-lang="en">Pending</span></span><strong>4</strong><small><span data-lang="zh-CN">仍需外部或人工证据接受。</span><span data-lang="en">External or human evidence still needs acceptance.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">已接受</span><span data-lang="en">Accepted</span></span><strong>0</strong><small><span data-lang="zh-CN">已通过 source check 与提交契约。</span><span data-lang="en">Passed source checks and submission contract.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">源检查</span><span data-lang="en">Source Checks</span></span><strong>9 / 19</strong><small><span data-lang="zh-CN">通过数 / 总检查数。</span><span data-lang="en">Passed checks / total checks.</span></small></article></div><div class='evidence-list'><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">提供商留出</span><span data-lang="en">Provider Holdout</span></h4></div><p><span data-lang="zh-CN">缺少真实 provider 模型运行和 token metadata。</span><span data-lang="en">Missing a real provider model run and token metadata.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">提供商实跑</span><span data-lang="en">Provider model run</span></li><li><span data-lang="zh-CN">Token 用量</span><span data-lang="en">Token usage observed</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">人工证据</span><span data-lang="en">Human evidence</span></span><h4><span data-lang="zh-CN">人工盲评</span><span data-lang="en">Human Adjudication</span></h4></div><p><span data-lang="zh-CN">盲评 pair 仍待真实 reviewer 决策。</span><span data-lang="en">Blind-review pairs still need real reviewer decisions.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">无待判定</span><span data-lang="en">No pending decisions</span></li><li><span data-lang="zh-CN">盲评完成</span><span data-lang="en">Judgments complete</span></li><li><span data-lang="zh-CN">Reviewer metadata</span><span data-lang="en">Reviewer metadata</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生权限</span><span data-lang="en">Native Permission</span></h4></div><p><span data-lang="zh-CN">原生 runtime enforcement 仍待目标客户端或外部安装器证明。</span><span data-lang="en">Native runtime enforcement still needs target-client or external-installer proof.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">原生执行</span><span data-lang="en">Native enforcement</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生遥测</span><span data-lang="en">Native Telemetry</span></h4></div><p><span data-lang="zh-CN">真实外部客户端 metadata-only 事件仍未导入。</span><span data-lang="en">Real external-client metadata-only events have not been imported yet.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">外部事件</span><span data-lang="en">External events</span></li><li><span data-lang="zh-CN">采用样本</span><span data-lang="en">Adoption sample</span></li></ul></article></div></article>
|
||||
<article class='panel world-readiness'><div class='world-readiness-head'><div><h3><span data-lang="zh-CN">世界证据</span><span data-lang="en">World Evidence</span></h3><p><span data-lang="zh-CN">世界级证据尚未完成：4 项待补，0 项已接受。</span><span data-lang="en">World-class evidence is not complete: 4 pending, 0 accepted.</span></p></div><span class='world-status'><span data-lang="zh-CN">证据待补</span><span data-lang="en">Evidence pending</span></span></div><div class='evidence-kpis'><article class='evidence-kpi'><span><span data-lang="zh-CN">待补证据</span><span data-lang="en">Pending</span></span><strong>4</strong><small><span data-lang="zh-CN">仍需外部或人工证据接受。</span><span data-lang="en">External or human evidence still needs acceptance.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">已接受</span><span data-lang="en">Accepted</span></span><strong>0</strong><small><span data-lang="zh-CN">已通过 source check 与提交契约。</span><span data-lang="en">Passed source checks and submission contract.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">源检查</span><span data-lang="en">Source Checks</span></span><strong>10 / 19</strong><small><span data-lang="zh-CN">通过数 / 总检查数。</span><span data-lang="en">Passed checks / total checks.</span></small></article></div><div class='evidence-list'><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">提供商留出</span><span data-lang="en">Provider Holdout</span></h4></div><p><span data-lang="zh-CN">缺少真实 provider 模型运行和 token metadata。</span><span data-lang="en">Missing a real provider model run and token metadata.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">提供商实跑</span><span data-lang="en">Provider model run</span></li><li><span data-lang="zh-CN">Token 用量</span><span data-lang="en">Token usage observed</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">人工证据</span><span data-lang="en">Human evidence</span></span><h4><span data-lang="zh-CN">人工盲评</span><span data-lang="en">Human Adjudication</span></h4></div><p><span data-lang="zh-CN">盲评 pair 仍待真实 reviewer 决策。</span><span data-lang="en">Blind-review pairs still need real reviewer decisions.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">无待判定</span><span data-lang="en">No pending decisions</span></li><li><span data-lang="zh-CN">盲评完成</span><span data-lang="en">Judgments complete</span></li><li><span data-lang="zh-CN">Reviewer metadata</span><span data-lang="en">Reviewer metadata</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生权限</span><span data-lang="en">Native Permission</span></h4></div><p><span data-lang="zh-CN">原生 runtime enforcement 仍待目标客户端或外部安装器证明。</span><span data-lang="en">Native runtime enforcement still needs target-client or external-installer proof.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">原生执行</span><span data-lang="en">Native enforcement</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生遥测</span><span data-lang="en">Native Telemetry</span></h4></div><p><span data-lang="zh-CN">真实外部客户端 metadata-only 事件仍未导入。</span><span data-lang="en">Real external-client metadata-only events have not been imported yet.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">外部事件</span><span data-lang="en">External events</span></li></ul></article></div></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -412,7 +412,7 @@
|
||||
"external_pending_count": 3,
|
||||
"human_pending_count": 1,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_pass_count": 10,
|
||||
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
|
||||
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
|
||||
"entries": [
|
||||
@@ -472,8 +472,7 @@
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
"blocked_checks": [
|
||||
"External events",
|
||||
"Adoption sample"
|
||||
"External events"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1010,12 +1009,12 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "9080619d1499c827ab337b9172cc889106cf64f8c5a8baa86badcc8fc42ecf1a",
|
||||
"source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"evidence_bundle_sha256": "48071ebca4a8bfc9b45e5d4194858870fd0e7f50faff88d4abb79308113126eb",
|
||||
"source_contract_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
@@ -1031,18 +1030,18 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 19,
|
||||
"world_class_source_pass_count": 9,
|
||||
"world_class_source_blocked_count": 10,
|
||||
"world_class_source_pass_count": 10,
|
||||
"world_class_source_blocked_count": 9,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"source_tree_dirty": false,
|
||||
"source_changed_file_count": 0,
|
||||
"generated_tree_dirty": false,
|
||||
"generated_changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 49,
|
||||
"source_tree_dirty": true,
|
||||
"source_changed_file_count": 5,
|
||||
"generated_tree_dirty": true,
|
||||
"generated_changed_file_count": 44
|
||||
},
|
||||
"commit": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"commit": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
|
||||
@@ -1147,7 +1146,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 244,
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1185,7 +1184,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -1317,7 +1316,7 @@
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"to": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1334,14 +1333,14 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -1350,7 +1349,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -1625,7 +1624,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1678,8 +1677,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_blocked_count": 10,
|
||||
"source_pass_count": 10,
|
||||
"source_blocked_count": 9,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -2105,7 +2104,7 @@
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -2142,7 +2141,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -2163,8 +2162,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -2180,8 +2179,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
"path": "evidence/world_class/submissions/native-client-telemetry.json",
|
||||
|
||||
@@ -929,7 +929,7 @@
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<article class='panel world-readiness'><div class='world-readiness-head'><div><h3><span data-lang="zh-CN">世界证据</span><span data-lang="en">World Evidence</span></h3><p><span data-lang="zh-CN">世界级证据尚未完成：4 项待补，0 项已接受。</span><span data-lang="en">World-class evidence is not complete: 4 pending, 0 accepted.</span></p></div><span class='world-status'><span data-lang="zh-CN">证据待补</span><span data-lang="en">Evidence pending</span></span></div><div class='evidence-kpis'><article class='evidence-kpi'><span><span data-lang="zh-CN">待补证据</span><span data-lang="en">Pending</span></span><strong>4</strong><small><span data-lang="zh-CN">仍需外部或人工证据接受。</span><span data-lang="en">External or human evidence still needs acceptance.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">已接受</span><span data-lang="en">Accepted</span></span><strong>0</strong><small><span data-lang="zh-CN">已通过 source check 与提交契约。</span><span data-lang="en">Passed source checks and submission contract.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">源检查</span><span data-lang="en">Source Checks</span></span><strong>9 / 19</strong><small><span data-lang="zh-CN">通过数 / 总检查数。</span><span data-lang="en">Passed checks / total checks.</span></small></article></div><div class='evidence-list'><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">提供商留出</span><span data-lang="en">Provider Holdout</span></h4></div><p><span data-lang="zh-CN">缺少真实 provider 模型运行和 token metadata。</span><span data-lang="en">Missing a real provider model run and token metadata.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">提供商实跑</span><span data-lang="en">Provider model run</span></li><li><span data-lang="zh-CN">Token 用量</span><span data-lang="en">Token usage observed</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">人工证据</span><span data-lang="en">Human evidence</span></span><h4><span data-lang="zh-CN">人工盲评</span><span data-lang="en">Human Adjudication</span></h4></div><p><span data-lang="zh-CN">盲评 pair 仍待真实 reviewer 决策。</span><span data-lang="en">Blind-review pairs still need real reviewer decisions.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">无待判定</span><span data-lang="en">No pending decisions</span></li><li><span data-lang="zh-CN">盲评完成</span><span data-lang="en">Judgments complete</span></li><li><span data-lang="zh-CN">Reviewer metadata</span><span data-lang="en">Reviewer metadata</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生权限</span><span data-lang="en">Native Permission</span></h4></div><p><span data-lang="zh-CN">原生 runtime enforcement 仍待目标客户端或外部安装器证明。</span><span data-lang="en">Native runtime enforcement still needs target-client or external-installer proof.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">原生执行</span><span data-lang="en">Native enforcement</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生遥测</span><span data-lang="en">Native Telemetry</span></h4></div><p><span data-lang="zh-CN">真实外部客户端 metadata-only 事件仍未导入。</span><span data-lang="en">Real external-client metadata-only events have not been imported yet.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">外部事件</span><span data-lang="en">External events</span></li><li><span data-lang="zh-CN">采用样本</span><span data-lang="en">Adoption sample</span></li></ul></article></div></article>
|
||||
<article class='panel world-readiness'><div class='world-readiness-head'><div><h3><span data-lang="zh-CN">世界证据</span><span data-lang="en">World Evidence</span></h3><p><span data-lang="zh-CN">世界级证据尚未完成：4 项待补，0 项已接受。</span><span data-lang="en">World-class evidence is not complete: 4 pending, 0 accepted.</span></p></div><span class='world-status'><span data-lang="zh-CN">证据待补</span><span data-lang="en">Evidence pending</span></span></div><div class='evidence-kpis'><article class='evidence-kpi'><span><span data-lang="zh-CN">待补证据</span><span data-lang="en">Pending</span></span><strong>4</strong><small><span data-lang="zh-CN">仍需外部或人工证据接受。</span><span data-lang="en">External or human evidence still needs acceptance.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">已接受</span><span data-lang="en">Accepted</span></span><strong>0</strong><small><span data-lang="zh-CN">已通过 source check 与提交契约。</span><span data-lang="en">Passed source checks and submission contract.</span></small></article><article class='evidence-kpi'><span><span data-lang="zh-CN">源检查</span><span data-lang="en">Source Checks</span></span><strong>10 / 19</strong><small><span data-lang="zh-CN">通过数 / 总检查数。</span><span data-lang="en">Passed checks / total checks.</span></small></article></div><div class='evidence-list'><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">提供商留出</span><span data-lang="en">Provider Holdout</span></h4></div><p><span data-lang="zh-CN">缺少真实 provider 模型运行和 token metadata。</span><span data-lang="en">Missing a real provider model run and token metadata.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">提供商实跑</span><span data-lang="en">Provider model run</span></li><li><span data-lang="zh-CN">Token 用量</span><span data-lang="en">Token usage observed</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">人工证据</span><span data-lang="en">Human evidence</span></span><h4><span data-lang="zh-CN">人工盲评</span><span data-lang="en">Human Adjudication</span></h4></div><p><span data-lang="zh-CN">盲评 pair 仍待真实 reviewer 决策。</span><span data-lang="en">Blind-review pairs still need real reviewer decisions.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">无待判定</span><span data-lang="en">No pending decisions</span></li><li><span data-lang="zh-CN">盲评完成</span><span data-lang="en">Judgments complete</span></li><li><span data-lang="zh-CN">Reviewer metadata</span><span data-lang="en">Reviewer metadata</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生权限</span><span data-lang="en">Native Permission</span></h4></div><p><span data-lang="zh-CN">原生 runtime enforcement 仍待目标客户端或外部安装器证明。</span><span data-lang="en">Native runtime enforcement still needs target-client or external-installer proof.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">原生执行</span><span data-lang="en">Native enforcement</span></li></ul></article><article class='evidence-item'><div><span><span data-lang="zh-CN">外部证据</span><span data-lang="en">External evidence</span></span><h4><span data-lang="zh-CN">原生遥测</span><span data-lang="en">Native Telemetry</span></h4></div><p><span data-lang="zh-CN">真实外部客户端 metadata-only 事件仍未导入。</span><span data-lang="en">Real external-client metadata-only events have not been imported yet.</span></p><h5><span data-lang="zh-CN">阻塞检查</span><span data-lang="en">Blocked Checks</span></h5><ul class='blocked-checks'><li><span data-lang="zh-CN">外部事件</span><span data-lang="en">External events</span></li></ul></article></div></article>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
+32
-33
@@ -411,7 +411,7 @@
|
||||
"external_pending_count": 3,
|
||||
"human_pending_count": 1,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_pass_count": 10,
|
||||
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
|
||||
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
|
||||
"entries": [
|
||||
@@ -471,8 +471,7 @@
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
"blocked_checks": [
|
||||
"External events",
|
||||
"Adoption sample"
|
||||
"External events"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1005,12 +1004,12 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 25,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "9080619d1499c827ab337b9172cc889106cf64f8c5a8baa86badcc8fc42ecf1a",
|
||||
"source_contract_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"evidence_bundle_sha256": "48071ebca4a8bfc9b45e5d4194858870fd0e7f50faff88d4abb79308113126eb",
|
||||
"source_contract_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
@@ -1026,18 +1025,18 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"world_class_source_check_count": 19,
|
||||
"world_class_source_pass_count": 9,
|
||||
"world_class_source_blocked_count": 10,
|
||||
"world_class_source_pass_count": 10,
|
||||
"world_class_source_blocked_count": 9,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"source_tree_dirty": false,
|
||||
"source_changed_file_count": 0,
|
||||
"generated_tree_dirty": false,
|
||||
"generated_changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 49,
|
||||
"source_tree_dirty": true,
|
||||
"source_changed_file_count": 5,
|
||||
"generated_tree_dirty": true,
|
||||
"generated_changed_file_count": 44
|
||||
},
|
||||
"commit": "136654cda911836efd177294a17efc4e5bf08ae9",
|
||||
"commit": "6ece6407e81e352af8dffbb0933d8ee5128a92c3",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
|
||||
@@ -1142,7 +1141,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 244,
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1180,7 +1179,7 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d",
|
||||
"package_sha256": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f",
|
||||
"archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2"
|
||||
},
|
||||
"compatibility": {
|
||||
@@ -1312,7 +1311,7 @@
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"to": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1329,14 +1328,14 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -1345,7 +1344,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -1620,7 +1619,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1673,8 +1672,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_blocked_count": 10,
|
||||
"source_pass_count": 10,
|
||||
"source_blocked_count": 9,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -2100,7 +2099,7 @@
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -2137,7 +2136,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -2158,8 +2157,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -2175,8 +2174,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
"path": "evidence/world_class/submissions/native-client-telemetry.json",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-16T20:58:28Z",
|
||||
"generated_at": "2026-06-16T21:09:22Z",
|
||||
"skill_dir": ".",
|
||||
"decision": "proposal-review",
|
||||
"source_supplied": false,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Daily SkillOps Report
|
||||
|
||||
Generated at: `2026-06-16T20:58:28Z`
|
||||
Generated at: `2026-06-16T21:09:22Z`
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-17",
|
||||
"generated_at": "2026-06-16T21:09:22Z",
|
||||
"skill_dir": ".",
|
||||
"decision": "curator-review",
|
||||
"week_id": "2026-W25",
|
||||
@@ -15,7 +15,7 @@
|
||||
"top_score": 88,
|
||||
"skill_count": 12,
|
||||
"actionable_portfolio_issue_count": 0,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4,
|
||||
@@ -35,7 +35,7 @@
|
||||
"top_score": 88,
|
||||
"skill_count": 12,
|
||||
"actionable_portfolio_issue_count": 0,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4,
|
||||
@@ -288,7 +288,7 @@
|
||||
"no_route_opportunity_count": 0
|
||||
},
|
||||
"release_state": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"evidence_consistency_ok": true,
|
||||
"public_world_class_ready": false,
|
||||
"world_class_pending_count": 4
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Weekly SkillOps Curator Report
|
||||
|
||||
Generated at: `2026-06-17`
|
||||
Generated at: `2026-06-16T21:09:22Z`
|
||||
Week: `2026-W25`
|
||||
|
||||
## Summary
|
||||
@@ -13,7 +13,7 @@ Week: `2026-W25`
|
||||
- top_score: `88`
|
||||
- skill_count: `12`
|
||||
- actionable_portfolio_issue_count: `0`
|
||||
- release_lock_ready: `True`
|
||||
- release_lock_ready: `False`
|
||||
- evidence_consistency_ok: `True`
|
||||
- public_world_class_ready: `False`
|
||||
- world_class_pending_count: `4`
|
||||
|
||||
@@ -75,7 +75,7 @@
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "5533575c144fb56486ab7320bfbd9f6655a6360cc1c2fbab837c40a7bc57ca6d"
|
||||
"to": "f57c75f37b2b029c9b7a42d18d8daa4b5c9bb51defbc8e22774583243473007f"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -328,7 +328,7 @@
|
||||
"source_accepted": false,
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_blocked_count": 10,
|
||||
"source_pass_count": 10,
|
||||
"source_blocked_count": 9,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -33,8 +33,8 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_blocked_count": 10,
|
||||
"source_pass_count": 10,
|
||||
"source_blocked_count": 9,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
@@ -486,7 +486,7 @@
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -523,7 +523,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -544,8 +544,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -561,8 +561,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"submission_state": {
|
||||
"status": "missing",
|
||||
"path": "evidence/world_class/submissions/native-client-telemetry.json",
|
||||
|
||||
@@ -8,8 +8,8 @@ Generated at: `2026-06-17`
|
||||
- ready to claim world-class: `false`
|
||||
- entries: `4`
|
||||
- source accepted: `0`
|
||||
- source checks: `9` pass / `19` total
|
||||
- source blocked: `10`
|
||||
- source checks: `10` pass / `19` total
|
||||
- source blocked: `9`
|
||||
- accepted: `0`
|
||||
- pending: `4`
|
||||
- human pending: `1`
|
||||
@@ -30,7 +30,7 @@ This ledger records the current evidence state. It requires both passing source
|
||||
| `provider-holdout` | `pending` | `missing` | `external` | model-executed 0; token-observed 0 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. |
|
||||
| `human-adjudication` | `pending` | `missing` | `human` | 0/5 decisions; pending 5 | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. |
|
||||
| `native-permission-enforcement` | `pending` | `missing` | `external` | native-enforced targets 0; installer-enforced targets 4 | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. |
|
||||
| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. |
|
||||
| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 1 | Install a real client against the native host and import production metadata-only events. |
|
||||
|
||||
## Provider Holdout
|
||||
|
||||
@@ -186,8 +186,8 @@ This ledger records the current evidence state. It requires both passing source
|
||||
|
||||
- objective: Import production metadata-only events from a real external client into the local drift loop.
|
||||
- source status: `external_required`
|
||||
- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}`
|
||||
- source checks: `1` pass / `3` total
|
||||
- observed state: `{"external_source_events": 0, "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false}`
|
||||
- source checks: `2` pass / `3` total
|
||||
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "ledger_counts_as_completion": false}`
|
||||
|
||||
### Provenance Requirements
|
||||
@@ -211,7 +211,7 @@ This ledger records the current evidence state. It requires both passing source
|
||||
| Check | Current | Expected | Status |
|
||||
| --- | --- | --- | --- |
|
||||
| External events | `0` | `>0` | `blocked` |
|
||||
| Adoption sample | `0` | `>0` | `blocked` |
|
||||
| Adoption sample | `1` | `>0` | `pass` |
|
||||
| Raw content blocked | `False` | `false` | `pass` |
|
||||
|
||||
### Completion Assertions
|
||||
|
||||
@@ -154,7 +154,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -325,7 +325,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
|
||||
@@ -22,7 +22,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It
|
||||
| `provider-holdout` | `external_required` | `external` | operator with provider credentials | model-executed 0; token-observed 0 |
|
||||
| `human-adjudication` | `human_required` | `human` | human reviewer | 0/5 decisions; pending 5 |
|
||||
| `native-permission-enforcement` | `external_required` | `external` | target client or installer integrator | native-enforced targets 0; installer-enforced targets 4 |
|
||||
| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 0 |
|
||||
| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 1 |
|
||||
|
||||
## Provider Holdout
|
||||
|
||||
|
||||
@@ -68,7 +68,7 @@
|
||||
<span class="eyebrow">Evidence Collection</span>
|
||||
<h1>World-Class Evidence Preflight</h1>
|
||||
<p class="lede">This operator view shows which external and human evidence is still blocked, which commands prepare editable submission drafts, and why preflight never counts as accepted evidence.</p>
|
||||
<div class="stats"><article><span>Decision</span><strong>collection-preflight-blocked</strong></article><article><span>Pending</span><strong>4</strong></article><article><span>Ready</span><strong>1</strong></article><article><span>Blocked</span><strong>3</strong></article><article><span>Source</span><strong>9/19</strong></article><article><span>Repairs</span><strong>14/14</strong></article><article><span>Phases</span><strong>2/2</strong></article></div>
|
||||
<div class="stats"><article><span>Decision</span><strong>collection-preflight-blocked</strong></article><article><span>Pending</span><strong>4</strong></article><article><span>Ready</span><strong>1</strong></article><article><span>Blocked</span><strong>3</strong></article><article><span>Source</span><strong>10/19</strong></article><article><span>Repairs</span><strong>13/13</strong></article><article><span>Phases</span><strong>2/2</strong></article></div>
|
||||
</section>
|
||||
<section class="section two-col" id="handoff">
|
||||
<article class="panel">
|
||||
@@ -128,7 +128,7 @@
|
||||
</header>
|
||||
<dl>
|
||||
<dt>Phase</dt><dd><code>collect-source</code></dd>
|
||||
<dt>Rows</dt><dd>10 / 10 blocked</dd>
|
||||
<dt>Rows</dt><dd>9 / 9 blocked</dd>
|
||||
<dt>Owners</dt><dd>Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator</dd>
|
||||
<dt>Evidence</dt><dd>human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout</dd>
|
||||
<dt>Next</dt><dd>Set reviewer_attestation only after choices are completed before opening the answer key.</dd>
|
||||
@@ -928,15 +928,15 @@
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<article class="check-row blocked">
|
||||
<article class="check-row pass">
|
||||
<div>
|
||||
<span>adoption_sample_count</span>
|
||||
<strong>Adoption sample</strong>
|
||||
</div>
|
||||
<dl>
|
||||
<dt>Current</dt><dd><code>0</code></dd>
|
||||
<dt>Current</dt><dd><code>1</code></dd>
|
||||
<dt>Expected</dt><dd><code>>0</code></dd>
|
||||
<dt>Status</dt><dd>blocked</dd>
|
||||
<dt>Status</dt><dd>pass</dd>
|
||||
<dt>Action</dt><dd>Telemetry must include adoption outcome evidence.</dd>
|
||||
</dl>
|
||||
</article>
|
||||
@@ -975,23 +975,6 @@
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<article class="repair-row blocked">
|
||||
<div>
|
||||
<span>source-check</span>
|
||||
<strong>adoption_sample_count</strong>
|
||||
</div>
|
||||
<dl>
|
||||
<dt>Priority</dt><dd>40</dd>
|
||||
<dt>Phase</dt><dd>collect-source</dd>
|
||||
<dt>Owner</dt><dd>Browser/Chrome/IDE/provider client integrator</dd>
|
||||
<dt>Status</dt><dd>blocked</dd>
|
||||
<dt>Reason</dt><dd>Current value 0 does not satisfy '>0'.</dd>
|
||||
<dt>Action</dt><dd>Telemetry must include adoption outcome evidence.</dd>
|
||||
<dt>Verify</dt><dd><code>python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions</code></dd>
|
||||
<dt>Evidence</dt><dd>does not count as completion</dd>
|
||||
</dl>
|
||||
</article>
|
||||
|
||||
<article class="repair-row blocked">
|
||||
<div>
|
||||
<span>source-check</span>
|
||||
|
||||
@@ -14,14 +14,14 @@
|
||||
"collection_ready_count": 1,
|
||||
"collection_blocked_count": 3,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_blocked_count": 10,
|
||||
"repair_checklist_count": 14,
|
||||
"repair_blocked_count": 14,
|
||||
"source_pass_count": 10,
|
||||
"source_blocked_count": 9,
|
||||
"repair_checklist_count": 13,
|
||||
"repair_blocked_count": 13,
|
||||
"repair_ready_count": 0,
|
||||
"repair_phase_counts": {
|
||||
"unblock-access": 4,
|
||||
"collect-source": 10
|
||||
"collect-source": 9
|
||||
},
|
||||
"next_repair_action_id": "human-adjudication-precheck-human-reviewer",
|
||||
"next_repair_phase": "unblock-access",
|
||||
@@ -30,7 +30,7 @@
|
||||
"repair_counts_as_completion": false,
|
||||
"phase_queue_count": 2,
|
||||
"phase_queue_blocked_count": 2,
|
||||
"phase_queue_row_count": 14,
|
||||
"phase_queue_row_count": 13,
|
||||
"phase_queue_next_phase": "unblock-access",
|
||||
"phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer",
|
||||
"phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
@@ -1076,8 +1076,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -1161,21 +1161,6 @@
|
||||
"source": "prechecks",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"action_id": "native-client-telemetry-source-check-adoption_sample_count",
|
||||
"evidence_key": "native-client-telemetry",
|
||||
"repair_type": "source-check",
|
||||
"target": "adoption_sample_count",
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "blocked",
|
||||
"blocking_reason": "Current value 0 does not satisfy '>0'.",
|
||||
"next_action": "Telemetry must include adoption outcome evidence.",
|
||||
"verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"source": "source_checklist",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"action_id": "native-client-telemetry-source-check-external_source_events",
|
||||
"evidence_key": "native-client-telemetry",
|
||||
@@ -1233,34 +1218,19 @@
|
||||
"label": "Collect source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 2,
|
||||
"row_count": 2,
|
||||
"blocked_count": 1,
|
||||
"row_count": 1,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator"
|
||||
],
|
||||
"evidence_keys": [
|
||||
"native-client-telemetry"
|
||||
],
|
||||
"next_action_id": "native-client-telemetry-source-check-adoption_sample_count",
|
||||
"next_action": "Telemetry must include adoption outcome evidence.",
|
||||
"next_action_id": "native-client-telemetry-source-check-external_source_events",
|
||||
"next_action": "Import at least one metadata-only event from a real client.",
|
||||
"verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"counts_as_completion": false,
|
||||
"rows": [
|
||||
{
|
||||
"action_id": "native-client-telemetry-source-check-adoption_sample_count",
|
||||
"evidence_key": "native-client-telemetry",
|
||||
"repair_type": "source-check",
|
||||
"target": "adoption_sample_count",
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "blocked",
|
||||
"blocking_reason": "Current value 0 does not satisfy '>0'.",
|
||||
"next_action": "Telemetry must include adoption outcome evidence.",
|
||||
"verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"source": "source_checklist",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"action_id": "native-client-telemetry-source-check-external_source_events",
|
||||
"evidence_key": "native-client-telemetry",
|
||||
@@ -1626,8 +1596,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -1778,21 +1748,6 @@
|
||||
"source": "source_checklist",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"action_id": "native-client-telemetry-source-check-adoption_sample_count",
|
||||
"evidence_key": "native-client-telemetry",
|
||||
"repair_type": "source-check",
|
||||
"target": "adoption_sample_count",
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "blocked",
|
||||
"blocking_reason": "Current value 0 does not satisfy '>0'.",
|
||||
"next_action": "Telemetry must include adoption outcome evidence.",
|
||||
"verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"source": "source_checklist",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"action_id": "native-client-telemetry-source-check-external_source_events",
|
||||
"evidence_key": "native-client-telemetry",
|
||||
@@ -1946,8 +1901,8 @@
|
||||
"label": "Collect source",
|
||||
"priority": 40,
|
||||
"status": "blocked",
|
||||
"blocked_count": 10,
|
||||
"row_count": 10,
|
||||
"blocked_count": 9,
|
||||
"row_count": 9,
|
||||
"owners": [
|
||||
"Browser/Chrome/IDE/provider client integrator",
|
||||
"human reviewer",
|
||||
@@ -2040,21 +1995,6 @@
|
||||
"source": "source_checklist",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"action_id": "native-client-telemetry-source-check-adoption_sample_count",
|
||||
"evidence_key": "native-client-telemetry",
|
||||
"repair_type": "source-check",
|
||||
"target": "adoption_sample_count",
|
||||
"phase": "collect-source",
|
||||
"priority": 40,
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "blocked",
|
||||
"blocking_reason": "Current value 0 does not satisfy '>0'.",
|
||||
"next_action": "Telemetry must include adoption outcome evidence.",
|
||||
"verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"source": "source_checklist",
|
||||
"counts_as_completion": false
|
||||
},
|
||||
{
|
||||
"action_id": "native-client-telemetry-source-check-external_source_events",
|
||||
"evidence_key": "native-client-telemetry",
|
||||
|
||||
@@ -10,10 +10,10 @@ Generated at: `2026-06-17`
|
||||
- credential value exposed: `false`
|
||||
- collection ready: `1`
|
||||
- collection blocked: `3`
|
||||
- source checks: `9` pass / `19` total
|
||||
- repair rows: `14` blocked / `14` total
|
||||
- source checks: `10` pass / `19` total
|
||||
- repair rows: `13` blocked / `13` total
|
||||
- phase queue: `2` blocked / `2` phases
|
||||
- phase queue rows: `14`
|
||||
- phase queue rows: `13`
|
||||
- next repair action: `human-adjudication-precheck-human-reviewer`
|
||||
- next repair owner: `human reviewer`
|
||||
- next phase: `unblock-access`
|
||||
@@ -51,7 +51,7 @@ Phase queue rows group the same repair checklist into operator execution phases.
|
||||
| Priority | Phase | Status | Rows | Owners | Evidence | Verify | Next action |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| `20` | `unblock-access` | `blocked` | 4 / 4 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Assign a real reviewer identity before claiming human adjudication. |
|
||||
| `40` | `collect-source` | `blocked` | 10 / 10 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. |
|
||||
| `40` | `collect-source` | `blocked` | 9 / 9 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. |
|
||||
|
||||
## Evidence Items
|
||||
|
||||
@@ -77,7 +77,6 @@ Repair rows convert preflight and source blockers into a prioritized operator qu
|
||||
| `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `pending_count` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Record a reviewer choice and reason for every pair. |
|
||||
| `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `ready_for_human_evidence` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints. |
|
||||
| `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `reviewer_metadata_present` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Record reviewer and reviewed_at before adjudication can count. |
|
||||
| `40` | `collect-source` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `source-check` | `adoption_sample_count` | `blocked` | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Telemetry must include adoption outcome evidence. |
|
||||
| `40` | `collect-source` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `source-check` | `external_source_events` | `blocked` | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Import at least one metadata-only event from a real client. |
|
||||
| `40` | `collect-source` | target client or installer integrator | `native-permission-enforcement` | `source-check` | `native_enforcement_count` | `blocked` | `python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Collect real target-client or external runtime guard proof. |
|
||||
| `40` | `collect-source` | operator with provider credentials | `provider-holdout` | `source-check` | `model_executed_count` | `blocked` | `python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60 && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Run provider-backed output-exec with real credentials. |
|
||||
@@ -193,7 +192,7 @@ Repair rows convert preflight and source blockers into a prioritized operator qu
|
||||
| Check | Current | Expected | Status | Next action |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. |
|
||||
| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. |
|
||||
| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. |
|
||||
| Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. |
|
||||
|
||||
## Boundary
|
||||
|
||||
@@ -57,9 +57,10 @@
|
||||
<span class="eyebrow">Evidence Operations</span>
|
||||
<h1>World-Class Operator Runbook</h1>
|
||||
<p class="lede">A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.</p>
|
||||
<div class="stats"><article><span>Pending</span><strong>4</strong></article><article><span>Awaiting</span><strong>4</strong></article><article><span>Ready</span><strong>0</strong></article><article><span>Source</span><strong>9/19</strong></article><article><span>Blocked</span><strong>10</strong></article><article><span>Invalid</span><strong>0</strong></article></div>
|
||||
<div class="stats"><article><span>Pending</span><strong>4</strong></article><article><span>Awaiting</span><strong>4</strong></article><article><span>Ready</span><strong>0</strong></article><article><span>Source</span><strong>10/19</strong></article><article><span>Queue</span><strong>2/2</strong></article><article><span>Blocked</span><strong>9</strong></article><article><span>Invalid</span><strong>0</strong></article></div>
|
||||
</section>
|
||||
<section class="section panel" id="fast-path"><h2>Fast Path</h2><ol><li>Run the real external or human work for one evidence item.</li><li>Generate and fill the matching submission draft.</li><li>Validate intake and inspect the submission review queue.</li><li>Refresh the ledger and run the claim guard before making any completion claim.</li></ol></section>
|
||||
<section class="section panel" id="phase-queue"><h2>Phase Queue</h2><ul class='source-checks'><li class='source-check'><span>Unblock access</span><code>unblock-access: 4 / 4</code><small>Assign a real reviewer identity before claiming human adjudication.</small></li><li class='source-check'><span>Collect source</span><code>collect-source: 9 / 9</code><small>Set reviewer_attestation only after choices are completed before opening the answer key.</small></li></ul></section>
|
||||
<section class="section" id="items"><h2>Evidence Items</h2><div class="item-grid"><article class="item-card awaiting-submission">
|
||||
<header><span>external · awaiting-submission</span><h3>Provider Holdout</h3></header>
|
||||
<p>No evidence packet has been submitted for review.</p>
|
||||
@@ -67,8 +68,10 @@
|
||||
<dt>Owner</dt><dd>operator with provider credentials</dd>
|
||||
<dt>Ledger</dt><dd><code>pending</code></dd>
|
||||
<dt>Blocked</dt><dd><code>2</code></dd>
|
||||
<dt>Queue</dt><dd><code>2</code></dd>
|
||||
<dt>Submission</dt><dd><code>evidence/world_class/submissions/provider-holdout.json</code></dd>
|
||||
</dl>
|
||||
<section class="source-panel"><h4>Phase Queue</h4><ul class='source-checks'><li class='source-check'><span>Unblock access</span><code>unblock-access: 1 / 1</code><small>Set OPENAI_API_KEY in the operator shell; never commit or print the value.</small></li><li class='source-check'><span>Collect source</span><code>collect-source: 2 / 2</code><small>Run provider-backed output-exec with real credentials.</small></li></ul></section>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul><li>Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.</li><li>export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}</li><li>python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul></section>
|
||||
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
|
||||
<div class="mini-grid">
|
||||
@@ -85,8 +88,10 @@
|
||||
<dt>Owner</dt><dd>human reviewer</dd>
|
||||
<dt>Ledger</dt><dd><code>pending</code></dd>
|
||||
<dt>Blocked</dt><dd><code>5</code></dd>
|
||||
<dt>Queue</dt><dd><code>2</code></dd>
|
||||
<dt>Submission</dt><dd><code>evidence/world_class/submissions/human-adjudication.json</code></dd>
|
||||
</dl>
|
||||
<section class="source-panel"><h4>Phase Queue</h4><ul class='source-checks'><li class='source-check'><span>Unblock access</span><code>unblock-access: 1 / 1</code><small>Assign a real reviewer identity before claiming human adjudication.</small></li><li class='source-check'><span>Collect source</span><code>collect-source: 5 / 5</code><small>Set reviewer_attestation only after choices are completed before opening the answer key.</small></li></ul></section>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul><li>python3 scripts/yao.py output-review-kit --write-template</li><li>Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.</li><li>python3 scripts/adjudicate_output_review.py --write-template</li><li>Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.</li><li>python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication</li><li>python3 scripts/yao.py output-review</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul></section>
|
||||
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
|
||||
<div class="mini-grid">
|
||||
@@ -103,8 +108,10 @@
|
||||
<dt>Owner</dt><dd>target client or installer integrator</dd>
|
||||
<dt>Ledger</dt><dd><code>pending</code></dd>
|
||||
<dt>Blocked</dt><dd><code>1</code></dd>
|
||||
<dt>Queue</dt><dd><code>2</code></dd>
|
||||
<dt>Submission</dt><dd><code>evidence/world_class/submissions/native-permission-enforcement.json</code></dd>
|
||||
</dl>
|
||||
<section class="source-panel"><h4>Phase Queue</h4><ul class='source-checks'><li class='source-check'><span>Unblock access</span><code>unblock-access: 1 / 1</code><small>Attach a real target-client or external installer runtime guard; metadata fallback is not enough.</small></li><li class='source-check'><span>Collect source</span><code>collect-source: 1 / 1</code><small>Collect real target-client or external runtime guard proof.</small></li></ul></section>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul><li>Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.</li><li>Update the generated target adapter only when the guard is actually enforced by that target.</li><li>python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip</li><li>python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation</li><li>python3 scripts/yao.py runtime-permissions . --package-dir dist</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul></section>
|
||||
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
|
||||
<div class="mini-grid">
|
||||
@@ -120,9 +127,11 @@
|
||||
<dl>
|
||||
<dt>Owner</dt><dd>Browser/Chrome/IDE/provider client integrator</dd>
|
||||
<dt>Ledger</dt><dd><code>pending</code></dd>
|
||||
<dt>Blocked</dt><dd><code>2</code></dd>
|
||||
<dt>Blocked</dt><dd><code>1</code></dd>
|
||||
<dt>Queue</dt><dd><code>2</code></dd>
|
||||
<dt>Submission</dt><dd><code>evidence/world_class/submissions/native-client-telemetry.json</code></dd>
|
||||
</dl>
|
||||
<section class="source-panel"><h4>Phase Queue</h4><ul class='source-checks'><li class='source-check'><span>Unblock access</span><code>unblock-access: 1 / 1</code><small>Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.</small></li><li class='source-check'><span>Collect source</span><code>collect-source: 1 / 1</code><small>Import at least one metadata-only event from a real client.</small></li></ul></section>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul><li>python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/</li><li>Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.</li><li>python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl</li><li>python3 scripts/yao.py skill-atlas --workspace-root .</li><li>python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD></li><li>Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.</li><li>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</li></ul></section>
|
||||
<section><h4>Commands</h4><ul class="commands"><li><span>prepare submission</span><code>python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions</code></li><li><span>validate intake</span><code>python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions</code></li><li><span>review queue</span><code>python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions</code></li><li><span>refresh ledger</span><code>python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions</code></li><li><span>guard claim</span><code>python3 scripts/yao.py world-class-claim-guard .</code></li></ul></section>
|
||||
<div class="mini-grid">
|
||||
@@ -130,8 +139,8 @@
|
||||
<section><h4>Success Checks</h4><ul><li>reports/adoption_drift_report.json summary.source_types.external > 0</li><li>reports/adoption_drift_report.json summary.adoption_sample_count > 0</li><li>reports/skill_os2_audit.json item native-client-telemetry status becomes pass</li></ul></section>
|
||||
<section><h4>Privacy</h4><ul><li>Telemetry must remain metadata-only and local-first.</li><li>Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.</li></ul></section>
|
||||
</div>
|
||||
<section class="source-panel"><h4>Next Source Actions</h4><ul><li>Import at least one metadata-only event from a real client.</li><li>Telemetry must include adoption outcome evidence.</li></ul></section>
|
||||
<section class="source-panel"><h4>Source Evidence Snapshot</h4><ul class='source-checks'><li class='source-check blocked'><span>External events</span><code>external_source_events: 0 / >0</code><small>Import at least one metadata-only event from a real client.</small></li><li class='source-check blocked'><span>Adoption sample</span><code>adoption_sample_count: 0 / >0</code><small>Telemetry must include adoption outcome evidence.</small></li><li class='source-check pass'><span>Raw content blocked</span><code>raw_content_allowed: False / false</code><small>Telemetry must stay metadata-only.</small></li></ul></section>
|
||||
<section class="source-panel"><h4>Next Source Actions</h4><ul><li>Import at least one metadata-only event from a real client.</li></ul></section>
|
||||
<section class="source-panel"><h4>Source Evidence Snapshot</h4><ul class='source-checks'><li class='source-check blocked'><span>External events</span><code>external_source_events: 0 / >0</code><small>Import at least one metadata-only event from a real client.</small></li><li class='source-check pass'><span>Adoption sample</span><code>adoption_sample_count: 1 / >0</code><small>Telemetry must include adoption outcome evidence.</small></li><li class='source-check pass'><span>Raw content blocked</span><code>raw_content_allowed: False / false</code><small>Telemetry must stay metadata-only.</small></li></ul></section>
|
||||
</article></div></section>
|
||||
<section class="section panel" id="boundary"><h2>Boundary</h2><ul><li>Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.</li><li>Valid intake means ready for submission review; ledger review still requires passing source evidence.</li><li>The world-class ledger and claim guard remain the source of truth.</li></ul></section>
|
||||
</main>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,6 +11,9 @@ Generated at: `2026-06-17`
|
||||
- pending: `4`
|
||||
- awaiting submission: `4`
|
||||
- ready for ledger review: `0`
|
||||
- phase queue: `2` blocked / `2` phases
|
||||
- phase queue rows: `13`
|
||||
- phase queue counts as completion: `false`
|
||||
|
||||
This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true.
|
||||
|
||||
@@ -21,6 +24,13 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
3. Replace template-only fields with aggregate evidence and provenance.
|
||||
4. Validate intake, review the queue, refresh the ledger, then run the claim guard.
|
||||
|
||||
## Phase Queue
|
||||
|
||||
| Phase | Status | Rows | Blocked | Owners | Next action | Verify |
|
||||
| --- | --- | ---: | ---: | --- | --- | --- |
|
||||
| `unblock-access` | `blocked` | `4` | `4` | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | Assign a real reviewer identity before claiming human adjudication. | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` |
|
||||
| `collect-source` | `blocked` | `9` | `9` | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | Set reviewer_attestation only after choices are completed before opening the answer key. | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` |
|
||||
|
||||
## Evidence Items
|
||||
|
||||
| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner |
|
||||
@@ -28,16 +38,25 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials. | operator with provider credentials |
|
||||
| `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | `5` | Record a reviewer choice and reason for every pair. | human reviewer |
|
||||
| `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Collect real target-client or external runtime guard proof. | target client or installer integrator |
|
||||
| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator |
|
||||
| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator |
|
||||
|
||||
## Provider Holdout
|
||||
|
||||
- objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.
|
||||
- blocking reason: No evidence packet has been submitted for review.
|
||||
- blocked source checks: `2`
|
||||
- repair rows: `3` blocked
|
||||
- phase queue: `2` blocked phases
|
||||
- submission: `evidence/world_class/submissions/provider-holdout.json`
|
||||
- template: `evidence/world_class/templates/provider-holdout.intake.json`
|
||||
|
||||
### Phase Queue
|
||||
|
||||
| Phase | Status | Rows | Blocked | Next action |
|
||||
| --- | --- | ---: | ---: | --- |
|
||||
| `unblock-access` | `blocked` | `1` | `1` | Set OPENAI_API_KEY in the operator shell; never commit or print the value. |
|
||||
| `collect-source` | `blocked` | `2` | `2` | Run provider-backed output-exec with real credentials. |
|
||||
|
||||
### Source Runbook
|
||||
|
||||
- Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
|
||||
@@ -101,9 +120,18 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
- objective: Record real blind A/B reviewer decisions before claiming human output review completion.
|
||||
- blocking reason: No evidence packet has been submitted for review.
|
||||
- blocked source checks: `5`
|
||||
- repair rows: `6` blocked
|
||||
- phase queue: `2` blocked phases
|
||||
- submission: `evidence/world_class/submissions/human-adjudication.json`
|
||||
- template: `evidence/world_class/templates/human-adjudication.intake.json`
|
||||
|
||||
### Phase Queue
|
||||
|
||||
| Phase | Status | Rows | Blocked | Next action |
|
||||
| --- | --- | ---: | ---: | --- |
|
||||
| `unblock-access` | `blocked` | `1` | `1` | Assign a real reviewer identity before claiming human adjudication. |
|
||||
| `collect-source` | `blocked` | `5` | `5` | Set reviewer_attestation only after choices are completed before opening the answer key. |
|
||||
|
||||
### Source Runbook
|
||||
|
||||
- python3 scripts/yao.py output-review-kit --write-template
|
||||
@@ -192,9 +220,18 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
- objective: Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.
|
||||
- blocking reason: No evidence packet has been submitted for review.
|
||||
- blocked source checks: `1`
|
||||
- repair rows: `2` blocked
|
||||
- phase queue: `2` blocked phases
|
||||
- submission: `evidence/world_class/submissions/native-permission-enforcement.json`
|
||||
- template: `evidence/world_class/templates/native-permission-enforcement.intake.json`
|
||||
|
||||
### Phase Queue
|
||||
|
||||
| Phase | Status | Rows | Blocked | Next action |
|
||||
| --- | --- | ---: | ---: | --- |
|
||||
| `unblock-access` | `blocked` | `1` | `1` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. |
|
||||
| `collect-source` | `blocked` | `1` | `1` | Collect real target-client or external runtime guard proof. |
|
||||
|
||||
### Source Runbook
|
||||
|
||||
- Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
|
||||
@@ -261,10 +298,19 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
|
||||
- objective: Import production metadata-only events from a real external client into the local drift loop.
|
||||
- blocking reason: No evidence packet has been submitted for review.
|
||||
- blocked source checks: `2`
|
||||
- blocked source checks: `1`
|
||||
- repair rows: `2` blocked
|
||||
- phase queue: `2` blocked phases
|
||||
- submission: `evidence/world_class/submissions/native-client-telemetry.json`
|
||||
- template: `evidence/world_class/templates/native-client-telemetry.intake.json`
|
||||
|
||||
### Phase Queue
|
||||
|
||||
| Phase | Status | Rows | Blocked | Next action |
|
||||
| --- | --- | ---: | ---: | --- |
|
||||
| `unblock-access` | `blocked` | `1` | `1` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. |
|
||||
| `collect-source` | `blocked` | `1` | `1` | Import at least one metadata-only event from a real client. |
|
||||
|
||||
### Source Runbook
|
||||
|
||||
- python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
|
||||
@@ -314,14 +360,13 @@ This runbook coordinates evidence collection only. It does not accept submission
|
||||
### Next Source Actions
|
||||
|
||||
- Import at least one metadata-only event from a real client.
|
||||
- Telemetry must include adoption outcome evidence.
|
||||
|
||||
### Source Evidence Snapshot
|
||||
|
||||
| Check | Current | Expected | Status | Next action |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. |
|
||||
| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. |
|
||||
| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. |
|
||||
| Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. |
|
||||
|
||||
## Boundary
|
||||
|
||||
@@ -14,8 +14,8 @@
|
||||
"unmatched_submission_count": 0,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_blocked_count": 10,
|
||||
"source_pass_count": 10,
|
||||
"source_blocked_count": 9,
|
||||
"review_counts_submission_as_completion": false,
|
||||
"summary": {
|
||||
"review_item_count": 4,
|
||||
@@ -27,8 +27,8 @@
|
||||
"unmatched_submission_count": 0,
|
||||
"invalid_submission_count": 0,
|
||||
"source_check_count": 19,
|
||||
"source_pass_count": 9,
|
||||
"source_blocked_count": 10,
|
||||
"source_pass_count": 10,
|
||||
"source_blocked_count": 9,
|
||||
"ready_to_claim_world_class": false,
|
||||
"review_counts_submission_as_completion": false,
|
||||
"decision": "awaiting-submissions"
|
||||
@@ -378,7 +378,7 @@
|
||||
"intake_errors": [],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -399,8 +399,8 @@
|
||||
"label": "Adoption sample",
|
||||
"field": "adoption_sample_count",
|
||||
"expected": ">0",
|
||||
"actual": 0,
|
||||
"status": "blocked",
|
||||
"actual": 1,
|
||||
"status": "pass",
|
||||
"source_accepted": false,
|
||||
"next_action": "Telemetry must include adoption outcome evidence."
|
||||
},
|
||||
@@ -416,8 +416,8 @@
|
||||
}
|
||||
],
|
||||
"source_check_count": 3,
|
||||
"source_pass_count": 1,
|
||||
"source_blocked_count": 2,
|
||||
"source_pass_count": 2,
|
||||
"source_blocked_count": 1,
|
||||
"success_checks": [
|
||||
"reports/adoption_drift_report.json summary.source_types.external > 0",
|
||||
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
|
||||
|
||||
@@ -153,7 +153,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w
|
||||
#### Source Checks
|
||||
|
||||
- External events: 0 / >0 => blocked
|
||||
- Adoption sample: 0 / >0 => blocked
|
||||
- Adoption sample: 1 / >0 => pass
|
||||
- Raw content blocked: False / false => pass
|
||||
|
||||
#### Completion Assertions
|
||||
|
||||
@@ -74,9 +74,27 @@ def keyed_phase_queue_signatures_from_items(items: dict[str, dict[str, Any]], qu
|
||||
return signatures
|
||||
|
||||
|
||||
def any_phase_queue_counts_as_completion(
|
||||
*queues: list[dict[str, Any]],
|
||||
item_maps: dict[str, dict[str, Any]] | None = None,
|
||||
) -> bool:
|
||||
item_maps = item_maps or {}
|
||||
for queue in queues:
|
||||
for item in queue:
|
||||
if isinstance(item, dict) and item.get("counts_as_completion") is True:
|
||||
return True
|
||||
for item in item_maps.values():
|
||||
queue = item.get("phase_queue", []) if isinstance(item.get("phase_queue", []), list) else []
|
||||
for row in queue:
|
||||
if isinstance(row, dict) and row.get("counts_as_completion") is True:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def build_phase_queue_consistency_check(
|
||||
*,
|
||||
world_class_preflight: dict[str, Any],
|
||||
world_class_operator_runbook: dict[str, Any],
|
||||
review_studio: dict[str, Any],
|
||||
report_paths: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
@@ -88,6 +106,7 @@ def build_phase_queue_consistency_check(
|
||||
expected_queue = build_phase_queue(repair_rows)
|
||||
expected_summary = summarize_phase_queue(expected_queue)
|
||||
preflight_items = keyed_preflight_items(world_class_preflight)
|
||||
operator_runbook_items = keyed_preflight_items(world_class_operator_runbook)
|
||||
review_steps = world_class_review_action_steps(review_studio)
|
||||
expected = {
|
||||
"summary": summary_signature(expected_summary),
|
||||
@@ -106,14 +125,35 @@ def build_phase_queue_consistency_check(
|
||||
if isinstance(world_class_preflight.get("phase_queue", []), list)
|
||||
else []
|
||||
),
|
||||
"operator_runbook_summary": summary_signature(
|
||||
world_class_operator_runbook.get("summary", {})
|
||||
if isinstance(world_class_operator_runbook.get("summary", {}), dict)
|
||||
else {}
|
||||
),
|
||||
"operator_runbook_top_level_phase_queue": phase_queue_signature(
|
||||
world_class_operator_runbook.get("phase_queue", [])
|
||||
if isinstance(world_class_operator_runbook.get("phase_queue", []), list)
|
||||
else []
|
||||
),
|
||||
"item_phase_queues": keyed_phase_queue_signatures_from_items(preflight_items, "phase_queue"),
|
||||
"operator_runbook_phase_queues": keyed_phase_queue_signatures_from_items(
|
||||
operator_runbook_items,
|
||||
"phase_queue",
|
||||
),
|
||||
"review_studio_phase_queues": keyed_phase_queue_signatures_from_items(review_steps, "phase_queue"),
|
||||
"phase_queue_counts_as_completion": any(
|
||||
item.get("counts_as_completion") is True
|
||||
for item in world_class_preflight.get("phase_queue", [])
|
||||
if isinstance(item, dict)
|
||||
"phase_queue_counts_as_completion": any_phase_queue_counts_as_completion(
|
||||
world_class_preflight.get("phase_queue", [])
|
||||
if isinstance(world_class_preflight.get("phase_queue", []), list)
|
||||
else [],
|
||||
world_class_operator_runbook.get("phase_queue", [])
|
||||
if isinstance(world_class_operator_runbook.get("phase_queue", []), list)
|
||||
else [],
|
||||
item_maps={**preflight_items, **operator_runbook_items, **review_steps},
|
||||
),
|
||||
}
|
||||
expected["operator_runbook_summary"] = expected["summary"]
|
||||
expected["operator_runbook_top_level_phase_queue"] = expected["top_level_phase_queue"]
|
||||
expected["operator_runbook_phase_queues"] = expected["item_phase_queues"]
|
||||
expected["review_studio_phase_queues"] = expected["item_phase_queues"]
|
||||
return {
|
||||
"key": "world-class-phase-queue-consistency",
|
||||
@@ -121,9 +161,13 @@ def build_phase_queue_consistency_check(
|
||||
"status": "pass" if expected == actual else "fail",
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
"paths": [report_paths["world_class_preflight"], report_paths["review_studio"]],
|
||||
"paths": [
|
||||
report_paths["world_class_preflight"],
|
||||
report_paths["world_class_operator_runbook"],
|
||||
report_paths["review_studio"],
|
||||
],
|
||||
"detail": (
|
||||
"Phase queues must be derived from repair rows in preflight and mirrored into Review Studio "
|
||||
"without counting queue guidance as completion evidence."
|
||||
"Phase queues must be derived from repair rows in preflight and mirrored into the operator runbook "
|
||||
"and Review Studio without counting queue guidance as completion evidence."
|
||||
),
|
||||
}
|
||||
|
||||
@@ -404,6 +404,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
checks.append(
|
||||
build_phase_queue_consistency_check(
|
||||
world_class_preflight=world_class_preflight,
|
||||
world_class_operator_runbook=world_class_operator_runbook,
|
||||
review_studio=review_studio,
|
||||
report_paths=REQUIRED_REPORTS,
|
||||
)
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any
|
||||
from html_rendering import html_text
|
||||
from render_world_class_evidence_intake import build_intake
|
||||
from render_world_class_evidence_ledger import build_ledger
|
||||
from render_world_class_preflight import build_preflight
|
||||
from render_world_class_submission_review import build_submission_review
|
||||
|
||||
|
||||
@@ -38,6 +39,7 @@ def build_runbook_item(
|
||||
entry: dict[str, Any],
|
||||
checklist: dict[str, Any],
|
||||
review_item: dict[str, Any],
|
||||
preflight_item: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
commands = checklist.get("commands", {}) if isinstance(checklist.get("commands", {}), dict) else {}
|
||||
must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {}
|
||||
@@ -82,6 +84,24 @@ def build_runbook_item(
|
||||
"source_checklist": source_checklist,
|
||||
"blocked_source_check_count": len(blocked_source_checks),
|
||||
"next_source_actions": next_source_actions,
|
||||
"repair_checklist": preflight_item.get("repair_checklist", [])
|
||||
if isinstance(preflight_item.get("repair_checklist", []), list)
|
||||
else [],
|
||||
"repair_blocked_count": sum(
|
||||
1
|
||||
for row in preflight_item.get("repair_checklist", [])
|
||||
if isinstance(row, dict) and row.get("status") != "ready"
|
||||
),
|
||||
"repair_counts_as_completion": False,
|
||||
"phase_queue": preflight_item.get("phase_queue", [])
|
||||
if isinstance(preflight_item.get("phase_queue", []), list)
|
||||
else [],
|
||||
"phase_queue_blocked_count": sum(
|
||||
1
|
||||
for row in preflight_item.get("phase_queue", [])
|
||||
if isinstance(row, dict) and row.get("status") != "ready"
|
||||
),
|
||||
"phase_queue_counts_as_completion": False,
|
||||
"submission_state": entry.get("submission_state", {}),
|
||||
"anti_overclaim": entry.get("anti_overclaim", {}),
|
||||
}
|
||||
@@ -92,14 +112,22 @@ def build_operator_runbook(skill_dir: Path, generated_at: str, submissions_dir:
|
||||
ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
review = build_submission_review(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
preflight = build_preflight(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
checklist_by_key = by_key(intake.get("operator_checklist", []), "evidence_key")
|
||||
review_by_key = by_key(review.get("items", []), "evidence_key")
|
||||
preflight_by_key = by_key(preflight.get("items", []), "evidence_key")
|
||||
items = [
|
||||
build_runbook_item(entry, checklist_by_key.get(str(entry.get("key", "")), {}), review_by_key.get(str(entry.get("key", "")), {}))
|
||||
build_runbook_item(
|
||||
entry,
|
||||
checklist_by_key.get(str(entry.get("key", "")), {}),
|
||||
review_by_key.get(str(entry.get("key", "")), {}),
|
||||
preflight_by_key.get(str(entry.get("key", "")), {}),
|
||||
)
|
||||
for entry in ledger.get("entries", [])
|
||||
]
|
||||
summary = ledger.get("summary", {})
|
||||
review_summary = review.get("summary", {})
|
||||
preflight_summary = preflight.get("summary", {}) if isinstance(preflight.get("summary", {}), dict) else {}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": True,
|
||||
@@ -116,6 +144,15 @@ def build_operator_runbook(skill_dir: Path, generated_at: str, submissions_dir:
|
||||
"source_check_count": review_summary.get("source_check_count", 0),
|
||||
"source_pass_count": review_summary.get("source_pass_count", 0),
|
||||
"source_blocked_count": review_summary.get("source_blocked_count", 0),
|
||||
"repair_checklist_count": preflight_summary.get("repair_checklist_count", 0),
|
||||
"repair_blocked_count": preflight_summary.get("repair_blocked_count", 0),
|
||||
"phase_queue_count": preflight_summary.get("phase_queue_count", 0),
|
||||
"phase_queue_blocked_count": preflight_summary.get("phase_queue_blocked_count", 0),
|
||||
"phase_queue_row_count": preflight_summary.get("phase_queue_row_count", 0),
|
||||
"phase_queue_next_phase": preflight_summary.get("phase_queue_next_phase", ""),
|
||||
"phase_queue_next_action_id": preflight_summary.get("phase_queue_next_action_id", ""),
|
||||
"phase_queue_next_command": preflight_summary.get("phase_queue_next_command", ""),
|
||||
"phase_queue_counts_as_completion": False,
|
||||
"ready_to_claim_world_class": summary.get("ready_to_claim_world_class") is True,
|
||||
"runbook_counts_as_completion": False,
|
||||
"decision": "ready-for-completion-audit" if summary.get("ready_to_claim_world_class") is True else "collect-evidence",
|
||||
@@ -125,9 +162,14 @@ def build_operator_runbook(skill_dir: Path, generated_at: str, submissions_dir:
|
||||
"runbook_counts_submission_as_completion": False,
|
||||
},
|
||||
"items": items,
|
||||
"repair_checklist": preflight.get("repair_checklist", [])
|
||||
if isinstance(preflight.get("repair_checklist", []), list)
|
||||
else [],
|
||||
"phase_queue": preflight.get("phase_queue", []) if isinstance(preflight.get("phase_queue", []), list) else [],
|
||||
"source_reports": {
|
||||
"ledger": "reports/world_class_evidence_ledger.json",
|
||||
"intake": "reports/world_class_evidence_intake.json",
|
||||
"preflight": "reports/world_class_evidence_preflight.json",
|
||||
"submission_review": "reports/world_class_submission_review.json",
|
||||
"claim_guard": "reports/world_class_claim_guard.json",
|
||||
},
|
||||
@@ -161,6 +203,9 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
f"- pending: `{summary['pending_count']}`",
|
||||
f"- awaiting submission: `{summary['awaiting_submission_count']}`",
|
||||
f"- ready for ledger review: `{summary['ready_for_ledger_review_count']}`",
|
||||
f"- phase queue: `{summary['phase_queue_blocked_count']}` blocked / `{summary['phase_queue_count']}` phases",
|
||||
f"- phase queue rows: `{summary['phase_queue_row_count']}`",
|
||||
f"- phase queue counts as completion: `{str(summary['phase_queue_counts_as_completion']).lower()}`",
|
||||
"",
|
||||
"This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true.",
|
||||
"",
|
||||
@@ -171,11 +216,30 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
"3. Replace template-only fields with aggregate evidence and provenance.",
|
||||
"4. Validate intake, review the queue, refresh the ledger, then run the claim guard.",
|
||||
"",
|
||||
"## Evidence Items",
|
||||
"",
|
||||
"| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner |",
|
||||
"| --- | --- | --- | --- | ---: | --- | --- |",
|
||||
]
|
||||
lines.extend(
|
||||
[
|
||||
"## Phase Queue",
|
||||
"",
|
||||
"| Phase | Status | Rows | Blocked | Owners | Next action | Verify |",
|
||||
"| --- | --- | ---: | ---: | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for row in report.get("phase_queue", []):
|
||||
owners = ", ".join(str(owner) for owner in row.get("owners", []))
|
||||
lines.append(
|
||||
f"| `{row.get('phase', '')}` | `{row.get('status', '')}` | `{row.get('row_count', 0)}` | "
|
||||
f"`{row.get('blocked_count', 0)}` | {owners} | {row.get('next_action', '')} | `{row.get('verification_command', '')}` |"
|
||||
)
|
||||
lines.append("")
|
||||
lines.extend(
|
||||
[
|
||||
"## Evidence Items",
|
||||
"",
|
||||
"| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner |",
|
||||
"| --- | --- | --- | --- | ---: | --- | --- |",
|
||||
]
|
||||
)
|
||||
for item in report["items"]:
|
||||
next_action = item.get("next_source_actions", ["none"])[0] if item.get("next_source_actions") else "none"
|
||||
lines.append(
|
||||
@@ -191,8 +255,28 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
f"- objective: {item['objective']}",
|
||||
f"- blocking reason: {item['blocking_reason']}",
|
||||
f"- blocked source checks: `{item.get('blocked_source_check_count', 0)}`",
|
||||
f"- repair rows: `{item.get('repair_blocked_count', 0)}` blocked",
|
||||
f"- phase queue: `{item.get('phase_queue_blocked_count', 0)}` blocked phases",
|
||||
f"- submission: `{item['submission_path'] or 'missing'}`",
|
||||
f"- template: `{item['template_path'] or 'missing'}`",
|
||||
"",
|
||||
"### Phase Queue",
|
||||
"",
|
||||
"| Phase | Status | Rows | Blocked | Next action |",
|
||||
"| --- | --- | ---: | ---: | --- |",
|
||||
]
|
||||
)
|
||||
item_phase_queue = item.get("phase_queue", [])
|
||||
if item_phase_queue:
|
||||
for row in item_phase_queue:
|
||||
lines.append(
|
||||
f"| `{row.get('phase', '')}` | `{row.get('status', '')}` | `{row.get('row_count', 0)}` | "
|
||||
f"`{row.get('blocked_count', 0)}` | {row.get('next_action', '')} |"
|
||||
)
|
||||
else:
|
||||
lines.append("| No phase queue listed. | `n/a` | `0` | `0` | n/a |")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"### Source Runbook",
|
||||
"",
|
||||
@@ -267,6 +351,21 @@ def html_source_checks(rows: list[dict[str, Any]]) -> str:
|
||||
return "<ul class='source-checks'>" + "".join(items) + "</ul>"
|
||||
|
||||
|
||||
def html_phase_queue(rows: list[dict[str, Any]]) -> str:
|
||||
if not rows:
|
||||
return "<p class='muted'>No phase queue listed.</p>"
|
||||
items = []
|
||||
for row in rows:
|
||||
items.append(
|
||||
"<li class='source-check'>"
|
||||
f"<span>{html_text(row.get('label', row.get('phase', '')))}</span>"
|
||||
f"<code>{html_text(row.get('phase', ''))}: {html_text(row.get('blocked_count', 0))} / {html_text(row.get('row_count', 0))}</code>"
|
||||
f"<small>{html_text(row.get('next_action', ''))}</small>"
|
||||
"</li>"
|
||||
)
|
||||
return "<ul class='source-checks'>" + "".join(items) + "</ul>"
|
||||
|
||||
|
||||
def render_html_item(item: dict[str, Any]) -> str:
|
||||
commands = "".join(
|
||||
f"<li><span>{html_text(label.replace('_', ' '))}</span><code>{html_text(command)}</code></li>"
|
||||
@@ -281,8 +380,10 @@ def render_html_item(item: dict[str, Any]) -> str:
|
||||
<dt>Owner</dt><dd>{html_text(item['owner'])}</dd>
|
||||
<dt>Ledger</dt><dd><code>{html_text(item['ledger_status'])}</code></dd>
|
||||
<dt>Blocked</dt><dd><code>{html_text(item.get('blocked_source_check_count', 0))}</code></dd>
|
||||
<dt>Queue</dt><dd><code>{html_text(item.get('phase_queue_blocked_count', 0))}</code></dd>
|
||||
<dt>Submission</dt><dd><code>{html_text(item['submission_path'])}</code></dd>
|
||||
</dl>
|
||||
<section class="source-panel"><h4>Phase Queue</h4>{html_phase_queue(item.get('phase_queue', []))}</section>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul>{html_list(item.get('execution_runbook', []), 'No source runbook listed.')}</ul></section>
|
||||
<section><h4>Commands</h4><ul class="commands">{commands}</ul></section>
|
||||
<div class="mini-grid">
|
||||
@@ -303,6 +404,7 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
("Awaiting", summary["awaiting_submission_count"]),
|
||||
("Ready", summary["ready_for_ledger_review_count"]),
|
||||
("Source", f"{summary.get('source_pass_count', 0)}/{summary.get('source_check_count', 0)}"),
|
||||
("Queue", f"{summary.get('phase_queue_blocked_count', 0)}/{summary.get('phase_queue_count', 0)}"),
|
||||
("Blocked", summary.get("source_blocked_count", 0)),
|
||||
("Invalid", summary["invalid_submission_count"]),
|
||||
]
|
||||
@@ -370,6 +472,7 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
<div class="stats">{stat_html}</div>
|
||||
</section>
|
||||
<section class="section panel" id="fast-path"><h2>Fast Path</h2><ol><li>Run the real external or human work for one evidence item.</li><li>Generate and fill the matching submission draft.</li><li>Validate intake and inspect the submission review queue.</li><li>Refresh the ledger and run the claim guard before making any completion claim.</li></ol></section>
|
||||
<section class="section panel" id="phase-queue"><h2>Phase Queue</h2>{html_phase_queue(report.get('phase_queue', []))}</section>
|
||||
<section class="section" id="items"><h2>Evidence Items</h2><div class="item-grid">{item_html}</div></section>
|
||||
<section class="section panel" id="boundary"><h2>Boundary</h2><ul><li>Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.</li><li>Valid intake means ready for submission review; ledger review still requires passing source evidence.</li><li>The world-class ledger and claim guard remain the source of truth.</li></ul></section>
|
||||
</main>
|
||||
|
||||
@@ -187,7 +187,11 @@ def main() -> None:
|
||||
]
|
||||
phase_queue_actual = checks["world-class-phase-queue-consistency"]["actual"]
|
||||
assert phase_queue_actual["summary"]["phase_queue_count"] == 2, phase_queue_actual
|
||||
assert phase_queue_actual["summary"]["phase_queue_row_count"] == 13, phase_queue_actual
|
||||
assert phase_queue_actual["summary"]["phase_queue_row_count"] >= 13, phase_queue_actual
|
||||
assert phase_queue_actual["summary"] == phase_queue_actual["operator_runbook_summary"], phase_queue_actual
|
||||
assert phase_queue_actual["top_level_phase_queue"] == phase_queue_actual[
|
||||
"operator_runbook_top_level_phase_queue"
|
||||
], phase_queue_actual
|
||||
assert phase_queue_actual["phase_queue_counts_as_completion"] is False, phase_queue_actual
|
||||
assert set(phase_queue_actual["item_phase_queues"]) == {
|
||||
"provider-holdout",
|
||||
@@ -195,6 +199,9 @@ def main() -> None:
|
||||
"native-permission-enforcement",
|
||||
"native-client-telemetry",
|
||||
}, phase_queue_actual
|
||||
assert phase_queue_actual["operator_runbook_phase_queues"] == phase_queue_actual["item_phase_queues"], (
|
||||
phase_queue_actual
|
||||
)
|
||||
assert phase_queue_actual["review_studio_phase_queues"] == phase_queue_actual["item_phase_queues"], (
|
||||
phase_queue_actual
|
||||
)
|
||||
@@ -361,6 +368,15 @@ def main() -> None:
|
||||
phase_queue_payload["phase_queue"][0]["counts_as_completion"] = True
|
||||
phase_queue_payload["items"][0]["phase_queue"][0]["row_count"] = 999
|
||||
phase_queue_path.write_text(json.dumps(phase_queue_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
operator_phase_queue_path = phase_queue_drift_root / "reports" / "world_class_operator_runbook.json"
|
||||
operator_phase_queue_payload = json.loads(operator_phase_queue_path.read_text(encoding="utf-8"))
|
||||
operator_phase_queue_payload["summary"]["phase_queue_count"] = 99
|
||||
operator_phase_queue_payload["phase_queue"][0]["row_count"] = 999
|
||||
operator_phase_queue_payload["items"][0]["phase_queue"][0]["row_count"] = 999
|
||||
operator_phase_queue_path.write_text(
|
||||
json.dumps(operator_phase_queue_payload, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
phase_queue_drift_proc = run(
|
||||
[
|
||||
sys.executable,
|
||||
|
||||
@@ -111,8 +111,19 @@ def main() -> None:
|
||||
assert summary["source_check_count"] >= 13, summary
|
||||
assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary
|
||||
assert summary["source_blocked_count"] >= 6, summary
|
||||
assert summary["repair_checklist_count"] >= summary["source_blocked_count"], summary
|
||||
assert summary["phase_queue_count"] == 2, summary
|
||||
assert summary["phase_queue_blocked_count"] == 2, summary
|
||||
assert summary["phase_queue_row_count"] == summary["repair_checklist_count"], summary
|
||||
assert summary["phase_queue_next_phase"] == "unblock-access", summary
|
||||
assert summary["phase_queue_counts_as_completion"] is False, summary
|
||||
assert summary["ready_to_claim_world_class"] is False, summary
|
||||
assert summary["runbook_counts_as_completion"] is False, summary
|
||||
assert payload["repair_checklist"], payload
|
||||
assert len(payload["phase_queue"]) == summary["phase_queue_count"], payload["phase_queue"]
|
||||
assert sum(item["row_count"] for item in payload["phase_queue"]) == summary["phase_queue_row_count"], payload[
|
||||
"phase_queue"
|
||||
]
|
||||
items = {item["evidence_key"]: item for item in payload["items"]}
|
||||
assert set(items) == {
|
||||
"provider-holdout",
|
||||
@@ -124,6 +135,11 @@ def main() -> None:
|
||||
assert provider["review_state"] == "awaiting-submission", provider
|
||||
assert provider["source_accepted"] is False, provider
|
||||
assert provider["blocked_source_check_count"] == 2, provider
|
||||
assert provider["repair_blocked_count"] == 3, provider
|
||||
assert provider["repair_counts_as_completion"] is False, provider
|
||||
assert provider["phase_queue_blocked_count"] == 2, provider
|
||||
assert provider["phase_queue_counts_as_completion"] is False, provider
|
||||
assert [item["phase"] for item in provider["phase_queue"]] == ["unblock-access", "collect-source"], provider
|
||||
assert any("output-exec --provider-runner openai" in step for step in provider["execution_runbook"]), provider
|
||||
assert not any("<redacted>" in step or "OPENAI_API_KEY=" in step for step in provider["execution_runbook"]), provider
|
||||
assert "Run provider-backed output-exec with real credentials." in provider["next_source_actions"], provider
|
||||
@@ -146,10 +162,14 @@ def main() -> None:
|
||||
markdown = output_md.read_text(encoding="utf-8")
|
||||
assert "World-Class Operator Runbook" in markdown, markdown
|
||||
assert "runbook counts as completion: `false`" in markdown, markdown
|
||||
assert "phase queue counts as completion: `false`" in markdown, markdown
|
||||
assert "## Phase Queue" in markdown, markdown
|
||||
assert "| `unblock-access` | `blocked` |" in markdown, markdown
|
||||
assert "Valid intake means ready for submission review; ledger review still requires passing source evidence." in markdown, markdown
|
||||
assert "| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner |" in markdown, markdown
|
||||
assert "| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials." in markdown, markdown
|
||||
assert "Source Runbook" in markdown, markdown
|
||||
assert "### Phase Queue" in markdown, markdown
|
||||
assert "output-exec --provider-runner openai" in markdown, markdown
|
||||
assert "<redacted>" not in markdown, markdown
|
||||
assert "OPENAI_API_KEY=<redacted>" not in markdown, markdown
|
||||
@@ -164,8 +184,11 @@ def main() -> None:
|
||||
assert "position:sticky" in html, html
|
||||
assert "<span>Ready</span><strong>0</strong>" in html, html
|
||||
assert "<span>Invalid</span><strong>0</strong>" in html, html
|
||||
assert f"<span>Queue</span><strong>{summary['phase_queue_blocked_count']}/{summary['phase_queue_count']}</strong>" in html, html
|
||||
assert f"<span>Blocked</span><strong>{summary['source_blocked_count']}</strong>" in html, html
|
||||
assert "<dt>Blocked</dt><dd><code>2</code></dd>" in html, html
|
||||
assert "<dt>Queue</dt><dd><code>2</code></dd>" in html, html
|
||||
assert "Phase Queue" in html, html
|
||||
assert "Next Source Actions" in html, html
|
||||
assert "Source Runbook" in html, html
|
||||
assert "output-exec --provider-runner openai" in html, html
|
||||
@@ -208,12 +231,16 @@ def main() -> None:
|
||||
assert submitted_summary["ready_for_ledger_review_count"] == 0, submitted_summary
|
||||
assert submitted_summary["source_pass_count"] + submitted_summary["source_blocked_count"] == submitted_summary["source_check_count"], submitted_summary
|
||||
assert submitted_summary["source_blocked_count"] >= 6, submitted_summary
|
||||
assert submitted_summary["phase_queue_count"] == 2, submitted_summary
|
||||
assert submitted_summary["phase_queue_blocked_count"] == 2, submitted_summary
|
||||
assert submitted_summary["phase_queue_counts_as_completion"] is False, submitted_summary
|
||||
assert submitted_summary["ready_to_claim_world_class"] is False, submitted_summary
|
||||
submitted_provider = {item["evidence_key"]: item for item in submitted["items"]}["provider-holdout"]
|
||||
assert submitted_provider["intake_readiness"] == "fix-submission", submitted_provider
|
||||
assert submitted_provider["review_state"] == "fix-submission", submitted_provider
|
||||
assert submitted_provider["source_accepted"] is False, submitted_provider
|
||||
assert submitted_provider["blocked_source_check_count"] == 2, submitted_provider
|
||||
assert submitted_provider["phase_queue_blocked_count"] == 2, submitted_provider
|
||||
assert submitted_provider["next_source_actions"] == provider["next_source_actions"], submitted_provider
|
||||
assert "tests/tmp_world_class_operator_runbook/valid_submissions" in submitted_provider["commands"]["validate_intake"], submitted_provider
|
||||
assert "tests/tmp_world_class_operator_runbook/valid_submissions" in submitted_provider["commands"]["review_queue"], submitted_provider
|
||||
|
||||
Reference in New Issue
Block a user