feat: require validated world-class evidence submissions
This commit is contained in:
yaojingang
@@ -76,6 +76,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept
|
||||
- `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete.
|
||||
- `scripts/render_world_class_evidence_ledger.py`: machine-checkable acceptance ledger for the remaining world-class evidence gaps. Keep anti-overclaim guards explicit so planned work, metadata fallbacks, pending review, and local command runners never count as final evidence.
|
||||
- `scripts/render_world_class_evidence_intake.py`: intake validator for external and human world-class evidence packets. Real submissions must reference concrete local aggregate artifacts with matching SHA-256 digests; templates may stay hash-free and must not count as evidence.
|
||||
- `scripts/world_class_evidence_contract.py`: shared intake contract and artifact-integrity validator. Keep ledger, intake, and submission review aligned so source evidence cannot be accepted without a valid real submission and matching artifact SHA-256 checks.
|
||||
- `scripts/render_world_class_submission_review.py`: read-only queue for external and human evidence packets after intake validation. Keep it from accepting evidence; it may only compare packet validity, source evidence checks, and ledger state.
|
||||
- `scripts/render_world_class_operator_runbook.py`: operator-facing world-class evidence runbook. Keep it as coordination guidance only; it must not accept evidence or flip world-class readiness.
|
||||
- `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable.
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"package_count": 1,
|
||||
"packages": [
|
||||
{
|
||||
@@ -16,7 +16,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c",
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -48,10 +48,10 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-14"
|
||||
"generated_at": "2026-06-13"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14T00:32:04Z",
|
||||
"generated_at": "2026-06-14T00:52:22Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
|
||||
@@ -4,10 +4,10 @@
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"python_file_count": 153,
|
||||
"script_file_count": 94,
|
||||
"python_file_count": 154,
|
||||
"script_file_count": 95,
|
||||
"test_file_count": 59,
|
||||
"internal_module_count": 19,
|
||||
"internal_module_count": 20,
|
||||
"cli_script_count": 76,
|
||||
"command_handler_count": 34,
|
||||
"warn_line_threshold": 900,
|
||||
|
||||
@@ -5,10 +5,10 @@ Generated at: `2026-06-13`
|
||||
## Summary
|
||||
|
||||
- decision: `pass`
|
||||
- python files: `153`
|
||||
- scripts: `94`
|
||||
- python files: `154`
|
||||
- scripts: `95`
|
||||
- tests: `59`
|
||||
- internal modules: `19`
|
||||
- internal modules: `20`
|
||||
- CLI scripts: `76`
|
||||
- Yao CLI command handlers: `34`
|
||||
- largest file lines: `899`
|
||||
|
||||
@@ -3,24 +3,24 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"skill_dir": ".",
|
||||
"commit": "bd7f1d8cccbebac1222c0ee7bbc01c1e3b991126",
|
||||
"commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": true,
|
||||
"changed_file_count": 27,
|
||||
"changed_file_count": 50,
|
||||
"sample": [
|
||||
" M AGENTS.md",
|
||||
" M evidence/world_class/README.md",
|
||||
" M evidence/world_class/intake.schema.json",
|
||||
" M registry/index.json",
|
||||
" M registry/packages/yao-meta-skill.json",
|
||||
" M reports/adoption_drift_report.json",
|
||||
" M reports/adoption_drift_report.md",
|
||||
" M reports/architecture_maintainability.json",
|
||||
" M reports/architecture_maintainability.md",
|
||||
" M reports/benchmark_reproducibility.json",
|
||||
" M reports/benchmark_reproducibility.md",
|
||||
" M reports/context_budget.json",
|
||||
" M reports/output_execution_runs.json",
|
||||
" M reports/output_execution_runs.md",
|
||||
" M reports/package_verification.json"
|
||||
" M reports/install_simulation.json",
|
||||
" M reports/install_simulation.md"
|
||||
]
|
||||
},
|
||||
"summary": {
|
||||
@@ -42,7 +42,7 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 27
|
||||
"changed_file_count": 50
|
||||
},
|
||||
"methodology": {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
@@ -116,7 +116,7 @@
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"exists": true,
|
||||
"bytes": 7967,
|
||||
"sha256": "9b77977fc455f5cc473317efb0a71ae07a13bb1ddc5874ba5acb296659c5bac0"
|
||||
"sha256": "0ff38df2c18b25b79b9bf36e3055908a57d920ff45fa697f8281ce3a960f61b5"
|
||||
},
|
||||
{
|
||||
"label": "blind_review",
|
||||
@@ -150,57 +150,57 @@
|
||||
"label": "trust_report",
|
||||
"path": "reports/security_trust_report.json",
|
||||
"exists": true,
|
||||
"bytes": 97005,
|
||||
"sha256": "dc188ff6756e94c207ebdd2e182dcb2432cf44d8f8f824c10ff31f6ba0a4078f"
|
||||
"bytes": 97621,
|
||||
"sha256": "3c95ea60b1c941d1adae1827ca0ec312140d72db4d45e512ace8636f75378ba2"
|
||||
},
|
||||
{
|
||||
"label": "python_compatibility",
|
||||
"path": "reports/python_compatibility.json",
|
||||
"exists": true,
|
||||
"bytes": 20367,
|
||||
"sha256": "3698aad5388cad659840d1d31dbc9c694e0067b1de1e84423336a34e22df9625"
|
||||
"bytes": 20499,
|
||||
"sha256": "4edb5722c888f2fe5fc6c3141535ab5d550b6b1c05e9b4230b77e6712a982fb4"
|
||||
},
|
||||
{
|
||||
"label": "registry_audit",
|
||||
"path": "reports/registry_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 3183,
|
||||
"sha256": "b9acd15b662da8835b0c2ba0ad554a409f3265c2c779753d68f8a8dc5fdd5820"
|
||||
"sha256": "ad7900024ed6cadd5ea717ba22e38f38402e1a883bff119da24f7ecfd92c7dd8"
|
||||
},
|
||||
{
|
||||
"label": "package_verification",
|
||||
"path": "reports/package_verification.json",
|
||||
"exists": true,
|
||||
"bytes": 19325,
|
||||
"sha256": "0e87470fb95411379ee79420db3c7069433d086378a69b78da1f4ad2bf45edd8"
|
||||
"sha256": "dd12856ce8306600e089b44018ae4f7fae66366fba60f9408da885fbf5ab8035"
|
||||
},
|
||||
{
|
||||
"label": "install_simulation",
|
||||
"path": "reports/install_simulation.json",
|
||||
"exists": true,
|
||||
"bytes": 8604,
|
||||
"sha256": "97055f63c699b2812bab1a0f05460f1065a6211164b4c22c9bebc8aab6ef012f"
|
||||
"bytes": 8758,
|
||||
"sha256": "aabe72c3f5bde128ff74a24aef3e35f0ad449a431b9a932e92ec4ae5f3f308e4"
|
||||
},
|
||||
{
|
||||
"label": "skill_os2_audit",
|
||||
"path": "reports/skill_os2_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 14309,
|
||||
"sha256": "9c7b86742ed4dd5c930fbc3572d943937a4817bbd465d89da12f0929911607d0"
|
||||
"sha256": "0c74deea906e75f6a116e2e3d043e134ce820bd12782af18428b3c04a02e6bd0"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_plan",
|
||||
"path": "reports/world_class_evidence_plan.json",
|
||||
"exists": true,
|
||||
"bytes": 10083,
|
||||
"sha256": "194d5a681920c3aca5e454d8839cdec551a076d54e360424831c5f312d1367b1"
|
||||
"bytes": 19532,
|
||||
"sha256": "a9bd392cd45f3accabe9b5befb9090456f383b1edbc323cab7d20930179dc229"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_ledger",
|
||||
"path": "reports/world_class_evidence_ledger.json",
|
||||
"exists": true,
|
||||
"bytes": 11308,
|
||||
"sha256": "dc4cf8cf18899bdf10d32fe72bb8f306ee297106e6cc25fd041d56e36631af63"
|
||||
"bytes": 11613,
|
||||
"sha256": "0bff14542475657a1261c1afcd87ffabc5d5c5e75bcd20afcfeacc44b16b7c14"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_intake",
|
||||
@@ -214,14 +214,14 @@
|
||||
"path": "reports/world_class_submission_review.json",
|
||||
"exists": true,
|
||||
"bytes": 7263,
|
||||
"sha256": "185a8cab25e7155ebd80cb66049c0965e4dc2d7cc38991f17464946f87fd26f2"
|
||||
"sha256": "4f03edb2ef28d9dfd27a393f8058d6e27d2b10fa8cbfe35f473e003556b9ffe0"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook",
|
||||
"path": "reports/world_class_operator_runbook.json",
|
||||
"exists": true,
|
||||
"bytes": 15002,
|
||||
"sha256": "3d643dc8170b50a37f91ffb190f69ad1e0d32f64914ebe2d731226728ac28cdc"
|
||||
"sha256": "614742ea666c64cdb68e10fc77022c31e68abe82ed88325aea21acb2f277063e"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook_markdown",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Benchmark Reproducibility
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Commit: `bd7f1d8cccbebac1222c0ee7bbc01c1e3b991126`
|
||||
Commit: `48f2826b8c9ebad3a9cc90e6166f4fd253f98308`
|
||||
Working tree dirty at generation: `true`
|
||||
|
||||
## Summary
|
||||
@@ -16,7 +16,7 @@ Working tree dirty at generation: `true`
|
||||
- provider evidence complete: `false`
|
||||
- human review complete: `false`
|
||||
- world-class ready: `false`
|
||||
- changed files at generation: `27`
|
||||
- changed files at generation: `50`
|
||||
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete.
|
||||
|
||||
@@ -40,22 +40,22 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
|
||||
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
|
||||
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `9b77977fc455` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `0ff38df2c18b` |
|
||||
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
|
||||
| review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` |
|
||||
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
|
||||
| runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `dc188ff6756e` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `3698aad5388c` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `b9acd15b662d` |
|
||||
| package_verification | `reports/package_verification.json` | present | `0e87470fb954` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `97055f63c699` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `9c7b86742ed4` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `194d5a681920` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `dc4cf8cf1889` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `3c95ea60b1c9` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `4edb5722c888` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `ad7900024ed6` |
|
||||
| package_verification | `reports/package_verification.json` | present | `dd12856ce830` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `aabe72c3f5bd` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `0c74deea906e` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `a9bd392cd45f` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `0bff14542475` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `2cb7e6beb565` |
|
||||
| world_class_submission_review | `reports/world_class_submission_review.json` | present | `185a8cab25e7` |
|
||||
| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `3d643dc8170b` |
|
||||
| world_class_submission_review | `reports/world_class_submission_review.json` | present | `4f03edb2ef28` |
|
||||
| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `614742ea666c` |
|
||||
| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `f67ca3d0aa9c` |
|
||||
| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `963026f551dc` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `250d616b028c` |
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"target_count": 5,
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 751,
|
||||
"other_text_tokens": 1178333,
|
||||
"other_text_tokens": 1183524,
|
||||
"estimated_initial_load_tokens": 944,
|
||||
"estimated_total_text_tokens": 1179084,
|
||||
"relevant_file_count": 498,
|
||||
"estimated_total_text_tokens": 1184275,
|
||||
"relevant_file_count": 500,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 130,
|
||||
"quality_density": 137.7
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"package_dir": "dist",
|
||||
"install_root": "dist/install-simulation/simulate-yao-meta-skill",
|
||||
"installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"package_dir": "tests/tmp_review_studio/dist",
|
||||
"install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill",
|
||||
"installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 573,
|
||||
"archive_entry_count": 574,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -26,7 +26,7 @@
|
||||
{
|
||||
"id": "archive-present",
|
||||
"status": "pass",
|
||||
"detail": "Package archive exists: dist/yao-meta-skill.zip"
|
||||
"detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip"
|
||||
},
|
||||
{
|
||||
"id": "archive-safe-paths",
|
||||
@@ -272,9 +272,9 @@
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
"artifacts": {
|
||||
"archive": "dist/yao-meta-skill.zip",
|
||||
"package_manifest": "dist/manifest.json",
|
||||
"json": "reports/install_simulation.json",
|
||||
"markdown": "reports/install_simulation.md"
|
||||
"archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip",
|
||||
"package_manifest": "tests/tmp_review_studio/dist/manifest.json",
|
||||
"json": "tests/tmp_review_studio/install_simulation.json",
|
||||
"markdown": "tests/tmp_review_studio/install_simulation.md"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Install Simulation
|
||||
|
||||
- OK: `True`
|
||||
- Package directory: `dist`
|
||||
- Package directory: `tests/tmp_review_studio/dist`
|
||||
- Archive extracted: `True`
|
||||
- Entrypoint loaded: `True`
|
||||
- Manifest loaded: `True`
|
||||
@@ -16,7 +16,7 @@
|
||||
|
||||
| Check | Status | Detail |
|
||||
| --- | --- | --- |
|
||||
| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip |
|
||||
| `archive-present` | `pass` | Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip |
|
||||
| `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries |
|
||||
| `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill |
|
||||
| `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable |
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.72,
|
||||
"duration_ms": 30.68,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -62,7 +62,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 23.92,
|
||||
"duration_ms": 28.53,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -85,7 +85,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.18,
|
||||
"duration_ms": 29.59,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -113,7 +113,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.49,
|
||||
"duration_ms": 31.84,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -136,7 +136,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.68,
|
||||
"duration_ms": 30.95,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -164,7 +164,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.46,
|
||||
"duration_ms": 27.6,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -187,7 +187,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.04,
|
||||
"duration_ms": 26.76,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -214,7 +214,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.01,
|
||||
"duration_ms": 26.87,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -237,7 +237,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 23.99,
|
||||
"duration_ms": 26.9,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -266,7 +266,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.34,
|
||||
"duration_ms": 27.3,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
|
||||
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
|
||||
|
||||
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
|
||||
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 24.72 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 23.92 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 24.18 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 25.49 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 24.68 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 24.46 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 24.04 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 24.01 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 23.99 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 24.34 | 98 | 100.0 | pass |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 30.68 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 28.53 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 29.59 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.84 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 30.95 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.6 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.76 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.87 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.9 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.3 | 98 | 100.0 | pass |
|
||||
|
||||
## Next Fixes
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_entry_count": 573,
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"archive_entry_count": 574,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
- Package directory: `dist`
|
||||
- Targets: `4 / 4` adapters present
|
||||
- Archive present: `True`
|
||||
- Archive SHA256: `8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e`
|
||||
- Archive SHA256: `76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe`
|
||||
- Failures: `0`
|
||||
- Warnings: `0`
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"root": ".",
|
||||
"summary": {
|
||||
"target_python": "3.11",
|
||||
"file_count": 156,
|
||||
"file_count": 157,
|
||||
"issue_count": 0,
|
||||
"syntax_error_count": 0,
|
||||
"fstring_311_violation_count": 0,
|
||||
@@ -562,6 +562,12 @@
|
||||
"issue_count": 0,
|
||||
"issues": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/world_class_evidence_contract.py",
|
||||
"ok": true,
|
||||
"issue_count": 0,
|
||||
"issues": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao.py",
|
||||
"ok": true,
|
||||
|
||||
@@ -6,7 +6,7 @@ Generated at: `2026-06-13`
|
||||
|
||||
- decision: `pass`
|
||||
- target python: `3.11`
|
||||
- files scanned: `156`
|
||||
- files scanned: `157`
|
||||
- issues: `0`
|
||||
- syntax errors: `0`
|
||||
- f-string 3.11 violations: `0`
|
||||
|
||||
@@ -21,8 +21,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c",
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -53,16 +53,16 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-14"
|
||||
"generated_at": "2026-06-13"
|
||||
},
|
||||
"index": {
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"package_count": 1,
|
||||
"packages": [
|
||||
{
|
||||
@@ -78,7 +78,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
- Maturity: `governed`
|
||||
- Owner: `Yao Team`
|
||||
- License: `MIT`
|
||||
- Package SHA256: `08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c`
|
||||
- Archive SHA256: `8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e`
|
||||
- Package SHA256: `c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb`
|
||||
- Archive SHA256: `76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe`
|
||||
- Install simulated: `True`
|
||||
|
||||
## Compatibility
|
||||
|
||||
@@ -434,12 +434,12 @@
|
||||
|
||||
<section>
|
||||
<h2>核心指标</h2>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review Kit</span><strong>0/5</strong><p>pending 5; answer key hidden</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Blueprint</span><strong>20/20</strong><p>2.0 coverage; evidence pending 4</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native; 4 installer-enforced</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>94 scripts scanned; secrets found</p></article><article class='metric'><span>Py Compat</span><strong>0</strong><p>156 files scanned for Python 3.11</p></article><article class='metric'><span>Arch Debt</span><strong>0</strong><p>899 largest lines; 34 CLI handlers</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Intake</span><strong>4/4</strong><p>0 valid submissions; 0 invalid</p></article><article class='metric'><span>Claim Guard</span><strong>0</strong><p>73 public surfaces scanned</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>573 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review Kit</span><strong>0/5</strong><p>pending 5; answer key hidden</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Blueprint</span><strong>20/20</strong><p>2.0 coverage; evidence pending 4</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native; 4 installer-enforced</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>95 scripts scanned; secrets found</p></article><article class='metric'><span>Py Compat</span><strong>0</strong><p>157 files scanned for Python 3.11</p></article><article class='metric'><span>Arch Debt</span><strong>0</strong><p>899 largest lines; 34 CLI handlers</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Intake</span><strong>4/4</strong><p>0 valid submissions; 0 invalid</p></article><article class='metric'><span>Claim Guard</span><strong>0</strong><p>73 public surfaces scanned</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>574 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>审查闸门</h2>
|
||||
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 94 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>Python 兼容</h3></div><p>Python 3.11; 156 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards</p><footer>reports/python_compatibility.json <a href='python_compatibility.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>架构维护</h3></div><p>153 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers</p><footer>reports/architecture_maintainability.json <a href='architecture_maintainability.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>世界证据</h3></div><p>4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true</p><footer>reports/world_class_evidence_ledger.json <a href='world_class_evidence_ledger.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
|
||||
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 95 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>Python 兼容</h3></div><p>Python 3.11; 157 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards</p><footer>reports/python_compatibility.json <a href='python_compatibility.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>架构维护</h3></div><p>154 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers</p><footer>reports/architecture_maintainability.json <a href='architecture_maintainability.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>世界证据</h3></div><p>4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true</p><footer>reports/world_class_evidence_ledger.json <a href='world_class_evidence_ledger.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
|
||||
</section>
|
||||
|
||||
<div class="twocol">
|
||||
@@ -506,12 +506,12 @@
|
||||
</section>
|
||||
|
||||
<section id="trust" class="twocol">
|
||||
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>94</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>95</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
<div class="panel"><h2>Python 兼容</h2><dl class='kv-grid'><div><dt>目标 Python</dt><dd>3.11</dd></div><div><dt>文件数</dt><dd>156</dd></div><div><dt>问题数</dt><dd>0</dd></div><div><dt>语法错误</dt><dd>0</dd></div><div><dt>F-string 3.11</dt><dd>0</dd></div></dl></div>
|
||||
<div class="panel"><h2>Python 兼容</h2><dl class='kv-grid'><div><dt>目标 Python</dt><dd>3.11</dd></div><div><dt>文件数</dt><dd>157</dd></div><div><dt>问题数</dt><dd>0</dd></div><div><dt>语法错误</dt><dd>0</dd></div><div><dt>F-string 3.11</dt><dd>0</dd></div></dl></div>
|
||||
<div class="panel"><h2>解释器边界</h2><p>CI 和发布审查以 Python 3.11 兼容为底线；本地更高版本允许的新语法不能绕过兼容门禁。</p></div>
|
||||
</section>
|
||||
|
||||
@@ -574,12 +574,12 @@
|
||||
|
||||
<section id="registry" class="twocol">
|
||||
<div class="panel"><h2>注册审计</h2><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p></div>
|
||||
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe</code></dd></div></dl></div>
|
||||
</section>
|
||||
|
||||
<section id="release" class="twocol">
|
||||
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p></div>
|
||||
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>573</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>574</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe</code></dd></div></dl></div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
|
||||
+124
-81
@@ -59,7 +59,7 @@
|
||||
"key": "trust-report",
|
||||
"label": "信任报告",
|
||||
"status": "pass",
|
||||
"detail": "0 secrets; 94 scripts; 3 network-capable scripts; 0 help smoke failures",
|
||||
"detail": "0 secrets; 95 scripts; 3 network-capable scripts; 0 help smoke failures",
|
||||
"evidence": "reports/security_trust_report.json",
|
||||
"link": "security_trust_report.md"
|
||||
},
|
||||
@@ -67,7 +67,7 @@
|
||||
"key": "python-compat",
|
||||
"label": "Python 兼容",
|
||||
"status": "pass",
|
||||
"detail": "Python 3.11; 156 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards",
|
||||
"detail": "Python 3.11; 157 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards",
|
||||
"evidence": "reports/python_compatibility.json",
|
||||
"link": "python_compatibility.md"
|
||||
},
|
||||
@@ -75,7 +75,7 @@
|
||||
"key": "architecture-maintainability",
|
||||
"label": "架构维护",
|
||||
"status": "pass",
|
||||
"detail": "153 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers",
|
||||
"detail": "154 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers",
|
||||
"evidence": "reports/architecture_maintainability.json",
|
||||
"link": "architecture_maintainability.md"
|
||||
},
|
||||
@@ -851,7 +851,7 @@
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 94
|
||||
"file_count": 95
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
@@ -866,7 +866,7 @@
|
||||
"file_count": 203
|
||||
}
|
||||
],
|
||||
"file_count": 361,
|
||||
"file_count": 362,
|
||||
"folder_count": 4,
|
||||
"distribution": [
|
||||
{
|
||||
@@ -891,7 +891,7 @@
|
||||
},
|
||||
{
|
||||
"label": "scripts",
|
||||
"value": 94
|
||||
"value": 95
|
||||
},
|
||||
{
|
||||
"label": "evals",
|
||||
@@ -1020,7 +1020,7 @@
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 94
|
||||
"file_count": 95
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
@@ -1349,9 +1349,9 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 45
|
||||
"changed_file_count": 50
|
||||
},
|
||||
"commit": "bd7f1d8cccbebac1222c0ee7bbc01c1e3b991126",
|
||||
"commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
@@ -1434,9 +1434,9 @@
|
||||
"failures": []
|
||||
},
|
||||
"trust_security": {
|
||||
"scanned_files": 178,
|
||||
"script_count": 94,
|
||||
"internal_module_count": 17,
|
||||
"scanned_files": 179,
|
||||
"script_count": 95,
|
||||
"internal_module_count": 18,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
"requirements-ci.txt"
|
||||
@@ -1454,8 +1454,8 @@
|
||||
"help_smoke_failed_count": 0,
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 178,
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"package_hash_file_count": 179,
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1493,8 +1493,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c",
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1525,12 +1525,12 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-14"
|
||||
"generated_at": "2026-06-13"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": []
|
||||
@@ -1541,8 +1541,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_entry_count": 573,
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"archive_entry_count": 574,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
@@ -1553,7 +1553,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 573,
|
||||
"archive_entry_count": 574,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -1620,12 +1620,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"to": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"to": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1745,7 +1745,10 @@
|
||||
"summary": {
|
||||
"audit_decision": "continue-iteration",
|
||||
"world_class_ready": false,
|
||||
"audit_world_class_ready": false,
|
||||
"ready_to_claim_world_class": false,
|
||||
"ledger_completion_required": true,
|
||||
"evidence_requirement_count": 4,
|
||||
"task_count": 4,
|
||||
"human_task_count": 1,
|
||||
"external_task_count": 3,
|
||||
@@ -1920,6 +1923,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"ledger_entry_count": 4,
|
||||
"source_accepted_count": 0,
|
||||
"accepted_count": 0,
|
||||
"pending_count": 4,
|
||||
"human_pending_count": 1,
|
||||
@@ -1928,6 +1932,7 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
"ready_to_claim_world_class": false,
|
||||
"decision": "evidence-pending"
|
||||
@@ -1940,6 +1945,7 @@
|
||||
"owner": "operator with provider credentials",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"provenance_requirements": [
|
||||
@@ -1995,6 +2001,7 @@
|
||||
"owner": "human reviewer",
|
||||
"status": "pending",
|
||||
"source_status": "human_required",
|
||||
"source_accepted": false,
|
||||
"current": "0/5 decisions; pending 5",
|
||||
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
|
||||
"provenance_requirements": [
|
||||
@@ -2054,6 +2061,7 @@
|
||||
"owner": "target client or installer integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "native-enforced targets 0; installer-enforced targets 4",
|
||||
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
|
||||
"provenance_requirements": [
|
||||
@@ -2116,6 +2124,7 @@
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"provenance_requirements": [
|
||||
@@ -2169,7 +2178,8 @@
|
||||
"source_plan": {
|
||||
"json": "reports/world_class_evidence_plan.json",
|
||||
"markdown": "reports/world_class_evidence_plan.md",
|
||||
"task_count": 4
|
||||
"task_count": 4,
|
||||
"evidence_requirement_count": 4
|
||||
}
|
||||
},
|
||||
"synthesis_highlights": [
|
||||
@@ -3794,7 +3804,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.72,
|
||||
"duration_ms": 28.38,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3822,7 +3832,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 23.92,
|
||||
"duration_ms": 27.21,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3845,7 +3855,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.18,
|
||||
"duration_ms": 27.7,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3873,7 +3883,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.49,
|
||||
"duration_ms": 27.35,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3896,7 +3906,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.68,
|
||||
"duration_ms": 28.38,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3924,7 +3934,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.46,
|
||||
"duration_ms": 28.65,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3947,7 +3957,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.04,
|
||||
"duration_ms": 28.6,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3974,7 +3984,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.01,
|
||||
"duration_ms": 27.75,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3997,7 +4007,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 23.99,
|
||||
"duration_ms": 27.86,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4026,7 +4036,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 24.34,
|
||||
"duration_ms": 27.32,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4866,7 +4876,7 @@
|
||||
"label": "Trust Security",
|
||||
"status": "pass",
|
||||
"objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.",
|
||||
"current": "94 scripts; secrets 0; help failures 0",
|
||||
"current": "95 scripts; secrets 0; help failures 0",
|
||||
"command": "python3 scripts/yao.py trust .",
|
||||
"test": "python3 tests/verify_trust_check.py",
|
||||
"evidence": [
|
||||
@@ -4940,7 +4950,7 @@
|
||||
"label": "Registry Distribution",
|
||||
"status": "pass",
|
||||
"objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.",
|
||||
"current": "archive entries 573; install failures 0",
|
||||
"current": "archive entries 574; install failures 0",
|
||||
"command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .",
|
||||
"test": "python3 tests/verify_registry_audit.py",
|
||||
"evidence": [
|
||||
@@ -5361,7 +5371,7 @@
|
||||
"compiled_targets": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"target_count": 5,
|
||||
@@ -10221,9 +10231,9 @@
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"scanned_files": 178,
|
||||
"script_count": 94,
|
||||
"internal_module_count": 17,
|
||||
"scanned_files": 179,
|
||||
"script_count": 95,
|
||||
"internal_module_count": 18,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
"requirements-ci.txt"
|
||||
@@ -10241,8 +10251,8 @@
|
||||
"help_smoke_failed_count": 0,
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 178,
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"package_hash_file_count": 179,
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
@@ -11483,6 +11493,20 @@
|
||||
"network_urls": [],
|
||||
"network_hosts": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/world_class_evidence_contract.py",
|
||||
"interface": "internal-module",
|
||||
"interface_declared": true,
|
||||
"interface_reason": "Imported by world-class evidence reports to share intake validation and artifact integrity checks.",
|
||||
"has_argparse": false,
|
||||
"has_main_guard": false,
|
||||
"uses_input": false,
|
||||
"uses_network": false,
|
||||
"uses_file_write": false,
|
||||
"uses_subprocess": false,
|
||||
"network_urls": [],
|
||||
"network_hosts": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao.py",
|
||||
"interface": "cli",
|
||||
@@ -11616,7 +11640,7 @@
|
||||
"checked_count": 77,
|
||||
"passed_count": 77,
|
||||
"failed_count": 0,
|
||||
"skipped_count": 17,
|
||||
"skipped_count": 18,
|
||||
"failed_scripts": [],
|
||||
"results": [
|
||||
{
|
||||
@@ -12435,6 +12459,10 @@
|
||||
"path": "scripts/skill_report_model.py",
|
||||
"reason": "internal module"
|
||||
},
|
||||
{
|
||||
"path": "scripts/world_class_evidence_contract.py",
|
||||
"reason": "internal module"
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao_cli_config.py",
|
||||
"reason": "internal module"
|
||||
@@ -12644,7 +12672,7 @@
|
||||
"root": ".",
|
||||
"summary": {
|
||||
"target_python": "3.11",
|
||||
"file_count": 156,
|
||||
"file_count": 157,
|
||||
"issue_count": 0,
|
||||
"syntax_error_count": 0,
|
||||
"fstring_311_violation_count": 0,
|
||||
@@ -13201,6 +13229,12 @@
|
||||
"issue_count": 0,
|
||||
"issues": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/world_class_evidence_contract.py",
|
||||
"ok": true,
|
||||
"issue_count": 0,
|
||||
"issues": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao.py",
|
||||
"ok": true,
|
||||
@@ -13610,10 +13644,10 @@
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"python_file_count": 153,
|
||||
"script_file_count": 94,
|
||||
"python_file_count": 154,
|
||||
"script_file_count": 95,
|
||||
"test_file_count": 59,
|
||||
"internal_module_count": 19,
|
||||
"internal_module_count": 20,
|
||||
"cli_script_count": 76,
|
||||
"command_handler_count": 34,
|
||||
"warn_line_threshold": 900,
|
||||
@@ -13724,10 +13758,10 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 751,
|
||||
"other_text_tokens": 1178333,
|
||||
"other_text_tokens": 1183524,
|
||||
"estimated_initial_load_tokens": 944,
|
||||
"estimated_total_text_tokens": 1179084,
|
||||
"relevant_file_count": 498,
|
||||
"estimated_total_text_tokens": 1184275,
|
||||
"relevant_file_count": 500,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 130,
|
||||
"quality_density": 137.7
|
||||
@@ -14678,6 +14712,7 @@
|
||||
"scripts/upgrade_check.py",
|
||||
"scripts/validate_skill.py",
|
||||
"scripts/verify_package.py",
|
||||
"scripts/world_class_evidence_contract.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_create_commands.py",
|
||||
@@ -15580,7 +15615,7 @@
|
||||
"adoption_drift": {
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14T00:32:04Z",
|
||||
"generated_at": "2026-06-14T00:50:30Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
@@ -15768,10 +15803,11 @@
|
||||
"world_class_evidence_ledger": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"ledger_entry_count": 4,
|
||||
"source_accepted_count": 0,
|
||||
"accepted_count": 0,
|
||||
"pending_count": 4,
|
||||
"human_pending_count": 1,
|
||||
@@ -15780,6 +15816,7 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
"ready_to_claim_world_class": false,
|
||||
"decision": "evidence-pending"
|
||||
@@ -15792,6 +15829,7 @@
|
||||
"owner": "operator with provider credentials",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"provenance_requirements": [
|
||||
@@ -15847,6 +15885,7 @@
|
||||
"owner": "human reviewer",
|
||||
"status": "pending",
|
||||
"source_status": "human_required",
|
||||
"source_accepted": false,
|
||||
"current": "0/5 decisions; pending 5",
|
||||
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
|
||||
"provenance_requirements": [
|
||||
@@ -15906,6 +15945,7 @@
|
||||
"owner": "target client or installer integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "native-enforced targets 0; installer-enforced targets 4",
|
||||
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
|
||||
"provenance_requirements": [
|
||||
@@ -15968,6 +16008,7 @@
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"provenance_requirements": [
|
||||
@@ -16021,11 +16062,13 @@
|
||||
"source_plan": {
|
||||
"json": "reports/world_class_evidence_plan.json",
|
||||
"markdown": "reports/world_class_evidence_plan.md",
|
||||
"task_count": 4
|
||||
"task_count": 4,
|
||||
"evidence_requirement_count": 4
|
||||
},
|
||||
"submissions": {
|
||||
"directory": "evidence/world_class/submissions",
|
||||
"ledger_counts_submission_as_completion": false
|
||||
"ledger_counts_submission_as_completion": false,
|
||||
"source_pass_requires_valid_submission": true
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_evidence_ledger.json",
|
||||
@@ -16036,7 +16079,7 @@
|
||||
"world_class_evidence_intake": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"schema_present": true,
|
||||
@@ -16330,7 +16373,7 @@
|
||||
"world_class_submission_review": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"review_item_count": 4,
|
||||
@@ -16506,7 +16549,7 @@
|
||||
"world_class_operator_runbook": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"evidence_item_count": 4,
|
||||
@@ -16825,7 +16868,7 @@
|
||||
"world_class_claim_guard": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"ledger_ready_to_claim_world_class": false,
|
||||
@@ -17189,8 +17232,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c",
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -17221,16 +17264,16 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-14"
|
||||
"generated_at": "2026-06-13"
|
||||
},
|
||||
"index": {
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"package_count": 1,
|
||||
"packages": [
|
||||
{
|
||||
@@ -17246,7 +17289,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -17269,8 +17312,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_entry_count": 573,
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"archive_entry_count": 574,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
@@ -17928,14 +17971,14 @@
|
||||
"install_simulation": {
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"package_dir": "dist",
|
||||
"install_root": "dist/install-simulation/simulate-yao-meta-skill",
|
||||
"installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"package_dir": "tests/tmp_review_studio/dist",
|
||||
"install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill",
|
||||
"installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 573,
|
||||
"archive_entry_count": 574,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -17953,7 +17996,7 @@
|
||||
{
|
||||
"id": "archive-present",
|
||||
"status": "pass",
|
||||
"detail": "Package archive exists: dist/yao-meta-skill.zip"
|
||||
"detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip"
|
||||
},
|
||||
{
|
||||
"id": "archive-safe-paths",
|
||||
@@ -18199,16 +18242,16 @@
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
"artifacts": {
|
||||
"archive": "dist/yao-meta-skill.zip",
|
||||
"package_manifest": "dist/manifest.json",
|
||||
"json": "reports/install_simulation.json",
|
||||
"markdown": "reports/install_simulation.md"
|
||||
"archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip",
|
||||
"package_manifest": "tests/tmp_review_studio/dist/manifest.json",
|
||||
"json": "tests/tmp_review_studio/install_simulation.json",
|
||||
"markdown": "tests/tmp_review_studio/install_simulation.md"
|
||||
}
|
||||
},
|
||||
"upgrade_check": {
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"previous": {
|
||||
"name": "yao-meta-skill",
|
||||
"version": "1.0.0",
|
||||
@@ -18277,12 +18320,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"to": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"to": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -18297,8 +18340,8 @@
|
||||
"artifacts": {
|
||||
"previous_package": "registry/examples/yao-meta-skill-1.0.0.json",
|
||||
"current_package": "reports/registry_audit.json",
|
||||
"json": "reports/upgrade_check.json",
|
||||
"markdown": "reports/upgrade_check.md"
|
||||
"json": "tests/tmp_review_studio/upgrade_check.json",
|
||||
"markdown": "tests/tmp_review_studio/upgrade_check.md"
|
||||
}
|
||||
},
|
||||
"manifest": {
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"scanned_files": 178,
|
||||
"script_count": 94,
|
||||
"internal_module_count": 17,
|
||||
"scanned_files": 179,
|
||||
"script_count": 95,
|
||||
"internal_module_count": 18,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
"requirements-ci.txt"
|
||||
@@ -22,8 +22,8 @@
|
||||
"help_smoke_failed_count": 0,
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 178,
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"package_hash_file_count": 179,
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
@@ -1264,6 +1264,20 @@
|
||||
"network_urls": [],
|
||||
"network_hosts": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/world_class_evidence_contract.py",
|
||||
"interface": "internal-module",
|
||||
"interface_declared": true,
|
||||
"interface_reason": "Imported by world-class evidence reports to share intake validation and artifact integrity checks.",
|
||||
"has_argparse": false,
|
||||
"has_main_guard": false,
|
||||
"uses_input": false,
|
||||
"uses_network": false,
|
||||
"uses_file_write": false,
|
||||
"uses_subprocess": false,
|
||||
"network_urls": [],
|
||||
"network_hosts": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao.py",
|
||||
"interface": "cli",
|
||||
@@ -1397,7 +1411,7 @@
|
||||
"checked_count": 77,
|
||||
"passed_count": 77,
|
||||
"failed_count": 0,
|
||||
"skipped_count": 17,
|
||||
"skipped_count": 18,
|
||||
"failed_scripts": [],
|
||||
"results": [
|
||||
{
|
||||
@@ -2216,6 +2230,10 @@
|
||||
"path": "scripts/skill_report_model.py",
|
||||
"reason": "internal module"
|
||||
},
|
||||
{
|
||||
"path": "scripts/world_class_evidence_contract.py",
|
||||
"reason": "internal module"
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao_cli_config.py",
|
||||
"reason": "internal module"
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# Security Trust Report
|
||||
|
||||
- OK: `True`
|
||||
- Scanned files: `178`
|
||||
- Scripts: `94`
|
||||
- Internal script modules: `17`
|
||||
- Scanned files: `179`
|
||||
- Scripts: `95`
|
||||
- Internal script modules: `18`
|
||||
- Secret findings: `0`
|
||||
- Network-capable scripts: `3`
|
||||
- Network policy covered scripts: `3`
|
||||
@@ -15,8 +15,8 @@
|
||||
- CLI help smoke failures: `0`
|
||||
- Interactive scripts: `0`
|
||||
- Package hash scope: `source-contract-without-generated-reports`
|
||||
- Package hash files: `178`
|
||||
- Package SHA256: `08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c`
|
||||
- Package hash files: `179`
|
||||
- Package SHA256: `c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb`
|
||||
|
||||
## Failures
|
||||
|
||||
@@ -149,6 +149,7 @@
|
||||
| scripts/upgrade_check.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/validate_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. |
|
||||
| scripts/yao.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/yao_cli_config.py | internal-module | True | False | False | False | False | False | False | Imported by yao.py for CLI target maps and side-effect-free shaping helpers. |
|
||||
| scripts/yao_cli_create_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep skill creation and quickstart command handlers out of the CLI orchestrator. |
|
||||
|
||||
@@ -771,7 +771,7 @@
|
||||
<p><span data-lang="zh-CN">让 reviewer 快速确认关键文件、目录和资产分布。</span><span data-lang="en">Lets reviewers confirm key files, directories, and asset distribution quickly.</span></p>
|
||||
</div>
|
||||
<div class="two-col">
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-0.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-1.5" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-2.3" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="24.6 75.4" stroke-dashoffset="-3.1" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="72.3 27.7" stroke-dashoffset="-27.7" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">361项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-0.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-1.5" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-2.3" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="24.4 75.6" stroke-dashoffset="-3.1" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="72.5 27.5" stroke-dashoffset="-27.5" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">362项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<table>
|
||||
<thead><tr><th><span data-lang="zh-CN">路径</span><span data-lang="en">Path</span></th><th><span data-lang="zh-CN">作用</span><span data-lang="en">Role</span></th><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th></tr></thead>
|
||||
<tbody><tr><td>SKILL.md</td><td><span data-lang="zh-CN">Skill 入口文件</span><span data-lang="en">Skill entrypoint</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>README.md</td><td><span data-lang="zh-CN">人类可读使用说明</span><span data-lang="en">Human-readable usage guide</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>agents/interface.yaml</td><td><span data-lang="zh-CN">跨平台接口元数据</span><span data-lang="en">Neutral interface metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>manifest.json</td><td><span data-lang="zh-CN">生命周期与打包元数据</span><span data-lang="en">Lifecycle and portability metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>references</td><td><span data-lang="zh-CN">扩展指导与复用资料</span><span data-lang="en">Extended guidance and reusable notes</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>scripts</td><td><span data-lang="zh-CN">确定性脚本或本地工具</span><span data-lang="en">Deterministic helpers or local tooling</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>evals</td><td><span data-lang="zh-CN">触发与质量检查</span><span data-lang="en">Trigger and quality checks</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>reports</td><td><span data-lang="zh-CN">生成的证据与总结报告</span><span data-lang="en">Generated evidence and overview artifacts</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr></tbody>
|
||||
|
||||
+31
-21
@@ -437,7 +437,7 @@
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 94
|
||||
"file_count": 95
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
@@ -452,7 +452,7 @@
|
||||
"file_count": 203
|
||||
}
|
||||
],
|
||||
"file_count": 361,
|
||||
"file_count": 362,
|
||||
"folder_count": 4,
|
||||
"distribution": [
|
||||
{
|
||||
@@ -477,7 +477,7 @@
|
||||
},
|
||||
{
|
||||
"label": "scripts",
|
||||
"value": 94
|
||||
"value": 95
|
||||
},
|
||||
{
|
||||
"label": "evals",
|
||||
@@ -606,7 +606,7 @@
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 94
|
||||
"file_count": 95
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
@@ -935,9 +935,9 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 45
|
||||
"changed_file_count": 50
|
||||
},
|
||||
"commit": "bd7f1d8cccbebac1222c0ee7bbc01c1e3b991126",
|
||||
"commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
@@ -1020,9 +1020,9 @@
|
||||
"failures": []
|
||||
},
|
||||
"trust_security": {
|
||||
"scanned_files": 178,
|
||||
"script_count": 94,
|
||||
"internal_module_count": 17,
|
||||
"scanned_files": 179,
|
||||
"script_count": 95,
|
||||
"internal_module_count": 18,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
"requirements-ci.txt"
|
||||
@@ -1040,8 +1040,8 @@
|
||||
"help_smoke_failed_count": 0,
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 178,
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"package_hash_file_count": 179,
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1079,8 +1079,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c",
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1111,12 +1111,12 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
},
|
||||
"generated_at": "2026-06-14"
|
||||
"generated_at": "2026-06-13"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": []
|
||||
@@ -1127,8 +1127,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e",
|
||||
"archive_entry_count": 573,
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"archive_entry_count": 574,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
@@ -1139,7 +1139,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 573,
|
||||
"archive_entry_count": 574,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -1206,12 +1206,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"to": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"to": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1331,7 +1331,10 @@
|
||||
"summary": {
|
||||
"audit_decision": "continue-iteration",
|
||||
"world_class_ready": false,
|
||||
"audit_world_class_ready": false,
|
||||
"ready_to_claim_world_class": false,
|
||||
"ledger_completion_required": true,
|
||||
"evidence_requirement_count": 4,
|
||||
"task_count": 4,
|
||||
"human_task_count": 1,
|
||||
"external_task_count": 3,
|
||||
@@ -1506,6 +1509,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"ledger_entry_count": 4,
|
||||
"source_accepted_count": 0,
|
||||
"accepted_count": 0,
|
||||
"pending_count": 4,
|
||||
"human_pending_count": 1,
|
||||
@@ -1514,6 +1518,7 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
"ready_to_claim_world_class": false,
|
||||
"decision": "evidence-pending"
|
||||
@@ -1526,6 +1531,7 @@
|
||||
"owner": "operator with provider credentials",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"provenance_requirements": [
|
||||
@@ -1581,6 +1587,7 @@
|
||||
"owner": "human reviewer",
|
||||
"status": "pending",
|
||||
"source_status": "human_required",
|
||||
"source_accepted": false,
|
||||
"current": "0/5 decisions; pending 5",
|
||||
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
|
||||
"provenance_requirements": [
|
||||
@@ -1640,6 +1647,7 @@
|
||||
"owner": "target client or installer integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "native-enforced targets 0; installer-enforced targets 4",
|
||||
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
|
||||
"provenance_requirements": [
|
||||
@@ -1702,6 +1710,7 @@
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"provenance_requirements": [
|
||||
@@ -1755,7 +1764,8 @@
|
||||
"source_plan": {
|
||||
"json": "reports/world_class_evidence_plan.json",
|
||||
"markdown": "reports/world_class_evidence_plan.md",
|
||||
"task_count": 4
|
||||
"task_count": 4,
|
||||
"evidence_requirement_count": 4
|
||||
}
|
||||
},
|
||||
"synthesis_highlights": [
|
||||
|
||||
@@ -151,6 +151,7 @@
|
||||
"scripts/upgrade_check.py",
|
||||
"scripts/validate_skill.py",
|
||||
"scripts/verify_package.py",
|
||||
"scripts/world_class_evidence_contract.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_create_commands.py",
|
||||
|
||||
@@ -197,7 +197,7 @@
|
||||
"key": "trust-security",
|
||||
"label": "Trust Security",
|
||||
"status": "pass",
|
||||
"current": "secrets 0; scripts 94; help failures 0",
|
||||
"current": "secrets 0; scripts 95; help failures 0",
|
||||
"target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable",
|
||||
"evidence": [
|
||||
{
|
||||
@@ -285,7 +285,7 @@
|
||||
"key": "registry-distribution",
|
||||
"label": "Registry Distribution",
|
||||
"status": "pass",
|
||||
"current": "zip entries 573; install failures 0; permission failures 0",
|
||||
"current": "zip entries 574; install failures 0; permission failures 0",
|
||||
"target": "Package metadata, archive checksum, package verification, and install simulation pass",
|
||||
"evidence": [
|
||||
{
|
||||
|
||||
@@ -23,11 +23,11 @@ Generated at: `2026-06-13`
|
||||
| Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. |
|
||||
| Benchmark Reproducibility | pass | artifacts 24; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
|
||||
| Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. |
|
||||
| Trust Security | pass | secrets 0; scripts 94; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
|
||||
| Trust Security | pass | secrets 0; scripts 95; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
|
||||
| Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. |
|
||||
| Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. |
|
||||
| Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. |
|
||||
| Registry Distribution | pass | zip entries 573; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
|
||||
| Registry Distribution | pass | zip entries 574; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
|
||||
| Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. |
|
||||
| Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. |
|
||||
| Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. |
|
||||
|
||||
@@ -122,7 +122,7 @@
|
||||
"label": "Trust Security",
|
||||
"status": "pass",
|
||||
"objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.",
|
||||
"current": "94 scripts; secrets 0; help failures 0",
|
||||
"current": "95 scripts; secrets 0; help failures 0",
|
||||
"command": "python3 scripts/yao.py trust .",
|
||||
"test": "python3 tests/verify_trust_check.py",
|
||||
"evidence": [
|
||||
@@ -196,7 +196,7 @@
|
||||
"label": "Registry Distribution",
|
||||
"status": "pass",
|
||||
"objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.",
|
||||
"current": "archive entries 573; install failures 0",
|
||||
"current": "archive entries 574; install failures 0",
|
||||
"command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .",
|
||||
"test": "python3 tests/verify_registry_audit.py",
|
||||
"evidence": [
|
||||
|
||||
@@ -21,9 +21,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
|
||||
| Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` |
|
||||
| Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` |
|
||||
| Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` |
|
||||
| Trust Security | `pass` | 94 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
|
||||
| Trust Security | `pass` | 95 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
|
||||
| Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` |
|
||||
| Registry Distribution | `pass` | archive entries 573; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
|
||||
| Registry Distribution | `pass` | archive entries 574; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
|
||||
| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
|
||||
| Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` |
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"previous": {
|
||||
"name": "yao-meta-skill",
|
||||
"version": "1.0.0",
|
||||
@@ -70,12 +70,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e"
|
||||
"to": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c"
|
||||
"to": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -90,7 +90,7 @@
|
||||
"artifacts": {
|
||||
"previous_package": "registry/examples/yao-meta-skill-1.0.0.json",
|
||||
"current_package": "reports/registry_audit.json",
|
||||
"json": "reports/upgrade_check.json",
|
||||
"markdown": "reports/upgrade_check.md"
|
||||
"json": "tests/tmp_review_studio/upgrade_check.json",
|
||||
"markdown": "tests/tmp_review_studio/upgrade_check.md"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"ledger_ready_to_claim_world_class": false,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# World-Class Claim Guard
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Generated at: `2026-06-13`
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"schema_present": true,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# World-Class Evidence Intake
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Generated at: `2026-06-13`
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"ledger_entry_count": 4,
|
||||
"source_accepted_count": 0,
|
||||
"accepted_count": 0,
|
||||
"pending_count": 4,
|
||||
"human_pending_count": 1,
|
||||
@@ -13,6 +14,7 @@
|
||||
"missing_submission_count": 4,
|
||||
"invalid_submission_count": 0,
|
||||
"submitted_but_pending_count": 0,
|
||||
"source_accepted_without_valid_submission_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
"ready_to_claim_world_class": false,
|
||||
"decision": "evidence-pending"
|
||||
@@ -25,6 +27,7 @@
|
||||
"owner": "operator with provider credentials",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"provenance_requirements": [
|
||||
@@ -80,6 +83,7 @@
|
||||
"owner": "human reviewer",
|
||||
"status": "pending",
|
||||
"source_status": "human_required",
|
||||
"source_accepted": false,
|
||||
"current": "0/5 decisions; pending 5",
|
||||
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
|
||||
"provenance_requirements": [
|
||||
@@ -139,6 +143,7 @@
|
||||
"owner": "target client or installer integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "native-enforced targets 0; installer-enforced targets 4",
|
||||
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
|
||||
"provenance_requirements": [
|
||||
@@ -201,6 +206,7 @@
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"source_accepted": false,
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"provenance_requirements": [
|
||||
@@ -254,11 +260,13 @@
|
||||
"source_plan": {
|
||||
"json": "reports/world_class_evidence_plan.json",
|
||||
"markdown": "reports/world_class_evidence_plan.md",
|
||||
"task_count": 4
|
||||
"task_count": 4,
|
||||
"evidence_requirement_count": 4
|
||||
},
|
||||
"submissions": {
|
||||
"directory": "evidence/world_class/submissions",
|
||||
"ledger_counts_submission_as_completion": false
|
||||
"ledger_counts_submission_as_completion": false,
|
||||
"source_pass_requires_valid_submission": true
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_evidence_ledger.json",
|
||||
|
||||
@@ -1,22 +1,24 @@
|
||||
# World-Class Evidence Ledger
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Generated at: `2026-06-13`
|
||||
|
||||
## Summary
|
||||
|
||||
- decision: `evidence-pending`
|
||||
- ready to claim world-class: `false`
|
||||
- entries: `4`
|
||||
- source accepted: `0`
|
||||
- accepted: `0`
|
||||
- pending: `4`
|
||||
- human pending: `1`
|
||||
- external pending: `3`
|
||||
- submitted entries: `0`
|
||||
- submitted but pending: `0`
|
||||
- source accepted without valid submission: `0`
|
||||
- invalid submissions: `0`
|
||||
- overclaim guard active: `true`
|
||||
|
||||
This ledger records the current evidence state. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.
|
||||
This ledger records the current evidence state. It requires both passing source evidence and a validated intake submission with artifact SHA-256 checks before accepting an item. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.
|
||||
|
||||
## Ledger
|
||||
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"audit_decision": "continue-iteration",
|
||||
"world_class_ready": false,
|
||||
"audit_world_class_ready": false,
|
||||
"ready_to_claim_world_class": false,
|
||||
"ledger_completion_required": true,
|
||||
"evidence_requirement_count": 4,
|
||||
"task_count": 4,
|
||||
"human_task_count": 1,
|
||||
"external_task_count": 3,
|
||||
@@ -171,6 +174,164 @@
|
||||
"audit_next_action": "Install a real client against the native host and import production metadata-only events."
|
||||
}
|
||||
],
|
||||
"evidence_requirements": [
|
||||
{
|
||||
"key": "provider-holdout",
|
||||
"label": "Provider Holdout",
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "operator with provider credentials",
|
||||
"current": "model-executed 0; token-observed 0",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake ."
|
||||
],
|
||||
"success_checks": [
|
||||
"reports/output_execution_runs.json summary.model_executed_count > 0",
|
||||
"reports/output_execution_runs.json summary.timing_observed_count > 0",
|
||||
"reports/output_execution_runs.json summary.token_observed_count > 0",
|
||||
"reports/skill_os2_audit.json item provider-holdout status becomes pass"
|
||||
],
|
||||
"evidence_artifacts": [
|
||||
"reports/output_execution_runs.json",
|
||||
"reports/output_execution_runs.md",
|
||||
"reports/skill_os2_audit.json",
|
||||
"evidence/world_class/intake.schema.json",
|
||||
"evidence/world_class/templates/provider-holdout.intake.json",
|
||||
"reports/world_class_evidence_intake.json",
|
||||
"reports/world_class_evidence_intake.md"
|
||||
],
|
||||
"privacy_contract": [
|
||||
"Do not commit provider credentials or environment dumps.",
|
||||
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts."
|
||||
],
|
||||
"audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence."
|
||||
},
|
||||
{
|
||||
"key": "human-adjudication",
|
||||
"label": "Human Adjudication",
|
||||
"status": "human_required",
|
||||
"category": "human",
|
||||
"owner": "human reviewer",
|
||||
"current": "0/5 decisions; pending 5",
|
||||
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
|
||||
"runbook": [
|
||||
"python3 scripts/yao.py output-review-kit --write-template",
|
||||
"Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.",
|
||||
"python3 scripts/adjudicate_output_review.py --write-template",
|
||||
"Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.",
|
||||
"python3 scripts/yao.py output-review",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake ."
|
||||
],
|
||||
"success_checks": [
|
||||
"reports/output_review_adjudication.json summary.pending_count == 0",
|
||||
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
|
||||
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
|
||||
"reports/skill_os2_audit.json item human-adjudication status becomes pass"
|
||||
],
|
||||
"evidence_artifacts": [
|
||||
"reports/output_blind_review_pack.md",
|
||||
"reports/output_review_kit.md",
|
||||
"reports/output_review_decisions.json",
|
||||
"reports/output_review_adjudication.json",
|
||||
"reports/output_review_adjudication.md",
|
||||
"evidence/world_class/intake.schema.json",
|
||||
"evidence/world_class/templates/human-adjudication.intake.json",
|
||||
"reports/world_class_evidence_intake.json",
|
||||
"reports/world_class_evidence_intake.md"
|
||||
],
|
||||
"privacy_contract": [
|
||||
"Reviewer decisions should not include raw user data or private customer detail.",
|
||||
"Keep the answer key separate until after decisions are recorded."
|
||||
],
|
||||
"audit_next_action": "Record real A/B choices in the decision template, then regenerate adjudication."
|
||||
},
|
||||
{
|
||||
"key": "native-permission-enforcement",
|
||||
"label": "Native Permission Enforcement",
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "target client or installer integrator",
|
||||
"current": "native-enforced targets 0; installer-enforced targets 4",
|
||||
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
|
||||
"runbook": [
|
||||
"Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.",
|
||||
"Update the generated target adapter only when the guard is actually enforced by that target.",
|
||||
"python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip",
|
||||
"python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation",
|
||||
"python3 scripts/yao.py runtime-permissions . --package-dir dist",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake ."
|
||||
],
|
||||
"success_checks": [
|
||||
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
|
||||
"reports/runtime_permission_probes.json summary.failure_count == 0",
|
||||
"reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence",
|
||||
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
|
||||
],
|
||||
"evidence_artifacts": [
|
||||
"dist/targets/*/adapter.json",
|
||||
"reports/runtime_permission_probes.json",
|
||||
"reports/runtime_permission_probes.md",
|
||||
"reports/install_simulation.json",
|
||||
"reports/install_simulation.md",
|
||||
"security/permission_policy.json",
|
||||
"evidence/world_class/intake.schema.json",
|
||||
"evidence/world_class/templates/native-permission-enforcement.intake.json",
|
||||
"reports/world_class_evidence_intake.json",
|
||||
"reports/world_class_evidence_intake.md"
|
||||
],
|
||||
"privacy_contract": [
|
||||
"Do not mark native_enforcement true for metadata-only fallbacks.",
|
||||
"Keep residual risks visible for targets that still rely on operator enforcement."
|
||||
],
|
||||
"audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement."
|
||||
},
|
||||
{
|
||||
"key": "native-client-telemetry",
|
||||
"label": "Native Client Telemetry",
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
"Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.",
|
||||
"python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl",
|
||||
"python3 scripts/yao.py skill-atlas --workspace-root .",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
"Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake ."
|
||||
],
|
||||
"success_checks": [
|
||||
"reports/adoption_drift_report.json summary.source_types.external > 0",
|
||||
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
|
||||
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass"
|
||||
],
|
||||
"evidence_artifacts": [
|
||||
"reports/adoption_drift_report.json",
|
||||
"reports/adoption_drift_report.md",
|
||||
"reports/telemetry_hook_recipes.json",
|
||||
"scripts/telemetry_native_host.py",
|
||||
"evidence/world_class/intake.schema.json",
|
||||
"evidence/world_class/templates/native-client-telemetry.intake.json",
|
||||
"reports/world_class_evidence_intake.json",
|
||||
"reports/world_class_evidence_intake.md"
|
||||
],
|
||||
"privacy_contract": [
|
||||
"Telemetry must remain metadata-only and local-first.",
|
||||
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field."
|
||||
],
|
||||
"audit_next_action": "Install a real client against the native host and import production metadata-only events."
|
||||
}
|
||||
],
|
||||
"source_audit": {
|
||||
"json": "reports/skill_os2_audit.json",
|
||||
"markdown": "reports/skill_os2_audit.md",
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
# World-Class Evidence Plan
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Generated at: `2026-06-13`
|
||||
|
||||
## Summary
|
||||
|
||||
- decision: `collect-external-evidence`
|
||||
- audit decision: `continue-iteration`
|
||||
- ready to claim world-class: `false`
|
||||
- ledger completion required: `true`
|
||||
- evidence requirements: `4`
|
||||
- tasks: `4`
|
||||
- human tasks: `1`
|
||||
- external tasks: `3`
|
||||
|
||||
This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan as completion.
|
||||
This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan or source-report pass as completion; the ledger must still validate accepted submissions.
|
||||
|
||||
## Task Table
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"evidence_item_count": 4,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# World-Class Operator Runbook
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Generated at: `2026-06-13`
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"review_item_count": 4,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# World-Class Submission Review
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Generated at: `2026-06-13`
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -1,255 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shlex
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_world_class_evidence_ledger import build_ledger
|
||||
from world_class_evidence_contract import load_json, rel_path, validate_payload
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Validates world-class human and external evidence intake packets before ledger review."
|
||||
|
||||
REQUIRED_TOP_LEVEL = [
|
||||
"schema_version",
|
||||
"evidence_key",
|
||||
"template_only",
|
||||
"category",
|
||||
"source_type",
|
||||
"submitted_by",
|
||||
"submitted_at",
|
||||
"summary",
|
||||
"artifact_refs",
|
||||
"provenance",
|
||||
"privacy",
|
||||
"anti_overclaim",
|
||||
"attestation",
|
||||
]
|
||||
REQUIRED_PRIVACY_FALSE = [
|
||||
"raw_user_content_included",
|
||||
"raw_provider_prompt_included",
|
||||
"credentials_included",
|
||||
"secrets_included",
|
||||
]
|
||||
REQUIRED_ANTI_OVERCLAIM_FALSE = [
|
||||
"planned_work_counts_as_evidence",
|
||||
"metadata_fallback_counts_as_native_enforcement",
|
||||
"pending_review_counts_as_human_decision",
|
||||
"local_command_runner_counts_as_provider_model",
|
||||
]
|
||||
REQUIRED_ATTESTATION_TRUE = [
|
||||
"real_external_or_human_evidence",
|
||||
"reviewer_or_operator_identity_present",
|
||||
"artifact_refs_reviewed",
|
||||
"privacy_contract_satisfied",
|
||||
]
|
||||
EXPECTED_SOURCE_TYPES = {
|
||||
"provider-holdout": "provider-output-eval",
|
||||
"human-adjudication": "blind-ab-review",
|
||||
"native-permission-enforcement": "runtime-permission-guard",
|
||||
"native-client-telemetry": "native-client-telemetry",
|
||||
}
|
||||
SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
|
||||
DISALLOWED_REAL_ARTIFACTS = {"reports/telemetry_events.jsonl"}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def add_error(errors: list[str], condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
errors.append(message)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def resolve_artifact_path(raw_path: str, root: Path) -> tuple[Path | None, str | None]:
|
||||
text = raw_path.strip()
|
||||
if any(token in text for token in ("<", ">", "*", "?")):
|
||||
return None, "artifact_refs path must be concrete, not a placeholder or glob"
|
||||
candidate = Path(text)
|
||||
if candidate.is_absolute():
|
||||
return None, "artifact_refs path must be relative to the skill directory"
|
||||
if ".." in candidate.parts:
|
||||
return None, "artifact_refs path must not escape the skill directory"
|
||||
resolved = (root / candidate).resolve()
|
||||
try:
|
||||
resolved.relative_to(root.resolve())
|
||||
except ValueError:
|
||||
return None, "artifact_refs path must stay inside the skill directory"
|
||||
return resolved, None
|
||||
|
||||
|
||||
def validate_artifact_refs(
|
||||
payload: dict[str, Any],
|
||||
errors: list[str],
|
||||
*,
|
||||
root: Path,
|
||||
template_expected: bool,
|
||||
) -> dict[str, int]:
|
||||
refs = payload.get("artifact_refs")
|
||||
add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference")
|
||||
stats = {
|
||||
"artifact_ref_count": len(refs) if isinstance(refs, list) else 0,
|
||||
"artifact_existing_count": 0,
|
||||
"artifact_sha256_verified_count": 0,
|
||||
}
|
||||
if not isinstance(refs, list):
|
||||
return stats
|
||||
for index, ref in enumerate(refs):
|
||||
if not isinstance(ref, dict):
|
||||
errors.append(f"artifact_refs[{index}] must be an object")
|
||||
continue
|
||||
path_text = str(ref.get("path", "")).strip()
|
||||
add_error(errors, bool(path_text), f"artifact_refs[{index}].path is required")
|
||||
add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required")
|
||||
add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content")
|
||||
if template_expected or not path_text:
|
||||
continue
|
||||
resolved, path_error = resolve_artifact_path(path_text, root)
|
||||
if path_error:
|
||||
errors.append(f"artifact_refs[{index}].path {path_error}")
|
||||
continue
|
||||
rel = rel_path(resolved, root)
|
||||
if rel in DISALLOWED_REAL_ARTIFACTS:
|
||||
errors.append(f"artifact_refs[{index}].path must not reference raw local telemetry logs")
|
||||
if not resolved.exists() or not resolved.is_file():
|
||||
errors.append(f"artifact_refs[{index}].path does not exist as a local file")
|
||||
continue
|
||||
stats["artifact_existing_count"] += 1
|
||||
declared = str(ref.get("sha256", "")).strip()
|
||||
if not declared:
|
||||
errors.append(f"artifact_refs[{index}].sha256 is required for a real submission")
|
||||
continue
|
||||
if not SHA256_RE.match(declared):
|
||||
errors.append(f"artifact_refs[{index}].sha256 must be a 64-character hex digest")
|
||||
continue
|
||||
actual = sha256_file(resolved)
|
||||
if actual.lower() != declared.lower():
|
||||
errors.append(f"artifact_refs[{index}].sha256 does not match local artifact")
|
||||
continue
|
||||
stats["artifact_sha256_verified_count"] += 1
|
||||
return stats
|
||||
|
||||
|
||||
def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None:
|
||||
key = str(payload.get("evidence_key", ""))
|
||||
provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
|
||||
if key == "provider-holdout":
|
||||
add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required")
|
||||
add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required")
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("credential_material_committed") is False,
|
||||
"provider-holdout must attest credential_material_committed is false",
|
||||
)
|
||||
elif key == "human-adjudication":
|
||||
add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required")
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("answer_key_opened_after_decisions") is True,
|
||||
"human-adjudication must attest answer_key_opened_after_decisions is true",
|
||||
)
|
||||
elif key == "native-permission-enforcement":
|
||||
add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required")
|
||||
add_error(
|
||||
errors,
|
||||
bool(str(provenance.get("guard_location", "")).strip()),
|
||||
"native-permission-enforcement provenance.guard_location is required",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
str(provenance.get("guard_scope", "")).strip()
|
||||
in {"target-client-native", "external-installer-runtime-guard"},
|
||||
"native-permission-enforcement provenance.guard_scope must be target-client-native or external-installer-runtime-guard",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("guard_blocks_undeclared_capability") is True,
|
||||
"native-permission-enforcement must attest guard_blocks_undeclared_capability is true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("metadata_fallback_retained_for_other_targets") is True,
|
||||
"native-permission-enforcement must retain metadata fallback for non-native targets",
|
||||
)
|
||||
elif key == "native-client-telemetry":
|
||||
add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required")
|
||||
add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external")
|
||||
add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only")
|
||||
|
||||
|
||||
def validate_payload(
|
||||
payload: dict[str, Any],
|
||||
entry: dict[str, Any],
|
||||
*,
|
||||
path: Path,
|
||||
root: Path,
|
||||
template_expected: bool,
|
||||
) -> dict[str, Any]:
|
||||
errors: list[str] = []
|
||||
for key in REQUIRED_TOP_LEVEL:
|
||||
add_error(errors, key in payload, f"missing {key}")
|
||||
evidence_key = str(entry.get("key", ""))
|
||||
add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0")
|
||||
add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}")
|
||||
add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}")
|
||||
add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}")
|
||||
add_error(
|
||||
errors,
|
||||
payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key),
|
||||
f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}",
|
||||
)
|
||||
add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required")
|
||||
add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required")
|
||||
add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required")
|
||||
artifact_integrity = validate_artifact_refs(payload, errors, root=root, template_expected=template_expected)
|
||||
privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {}
|
||||
for key in REQUIRED_PRIVACY_FALSE:
|
||||
add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false")
|
||||
anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {}
|
||||
for key in REQUIRED_ANTI_OVERCLAIM_FALSE:
|
||||
add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false")
|
||||
validate_evidence_specific(payload, errors)
|
||||
attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {}
|
||||
if not template_expected:
|
||||
for key in REQUIRED_ATTESTATION_TRUE:
|
||||
add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission")
|
||||
return {
|
||||
"path": rel_path(path, root),
|
||||
"evidence_key": evidence_key,
|
||||
"status": "pass" if not errors else "fail",
|
||||
"template_only": template_expected,
|
||||
"artifact_integrity": artifact_integrity,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def template_paths(skill_dir: Path, keys: list[str]) -> dict[str, Path]:
|
||||
template_dir = skill_dir / "evidence" / "world_class" / "templates"
|
||||
return {key: template_dir / f"{key}.intake.json" for key in keys}
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_world_class_evidence_plan import build_plan
|
||||
from world_class_evidence_contract import load_json, load_json_with_status, rel_path, validate_payload
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -13,35 +14,6 @@ SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a machine-checkable ledger for world-class external and human evidence gaps."
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_json_with_status(path: Path) -> tuple[dict[str, Any], str]:
|
||||
if not path.exists():
|
||||
return {}, "missing"
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}, "invalid-json"
|
||||
if not isinstance(payload, dict):
|
||||
return {}, "invalid-json"
|
||||
return payload, "present"
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def provider_state(skill_dir: Path) -> dict[str, Any]:
|
||||
summary = load_json(skill_dir / "reports" / "output_execution_runs.json").get("summary", {})
|
||||
return {
|
||||
@@ -120,7 +92,8 @@ PROVENANCE_REQUIREMENTS = {
|
||||
}
|
||||
|
||||
|
||||
def submission_state(skill_dir: Path, key: str, submissions_dir: Path) -> dict[str, Any]:
|
||||
def submission_state(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]:
|
||||
key = str(task.get("key", ""))
|
||||
path = submissions_dir / f"{key}.json"
|
||||
payload, load_status = load_json_with_status(path)
|
||||
if load_status != "present":
|
||||
@@ -132,32 +105,32 @@ def submission_state(skill_dir: Path, key: str, submissions_dir: Path) -> dict[s
|
||||
"privacy_contract_satisfied": False,
|
||||
"ledger_counts_as_completion": False,
|
||||
}
|
||||
errors = []
|
||||
if payload.get("evidence_key") != key:
|
||||
errors.append("evidence_key mismatch")
|
||||
if payload.get("template_only") is not False:
|
||||
errors.append("template_only must be false")
|
||||
validation = validate_payload(payload, task, path=path, root=skill_dir, template_expected=False)
|
||||
refs = payload.get("artifact_refs", [])
|
||||
attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {}
|
||||
status = "submitted" if not errors else "invalid-contract"
|
||||
status = "submitted" if validation["status"] == "pass" else "invalid-contract"
|
||||
artifact_integrity = validation.get("artifact_integrity", {})
|
||||
return {
|
||||
"status": status,
|
||||
"path": rel_path(path, skill_dir),
|
||||
"submitted_by": payload.get("submitted_by", ""),
|
||||
"submitted_at": payload.get("submitted_at", ""),
|
||||
"artifact_ref_count": len(refs) if isinstance(refs, list) else 0,
|
||||
"artifact_existing_count": artifact_integrity.get("artifact_existing_count", 0),
|
||||
"artifact_sha256_verified_count": artifact_integrity.get("artifact_sha256_verified_count", 0),
|
||||
"attested_real_evidence": attestation.get("real_external_or_human_evidence") is True,
|
||||
"reviewer_or_operator_identity_present": attestation.get("reviewer_or_operator_identity_present") is True,
|
||||
"privacy_contract_satisfied": attestation.get("privacy_contract_satisfied") is True,
|
||||
"errors": errors,
|
||||
"errors": validation.get("errors", []),
|
||||
"ledger_counts_as_completion": False,
|
||||
}
|
||||
|
||||
|
||||
def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]:
|
||||
state = STATE_LOADERS.get(task["key"], lambda _: {"accepted": task["status"] == "pass"})(skill_dir)
|
||||
accepted = bool(state.get("accepted"))
|
||||
submission = submission_state(skill_dir, task["key"], submissions_dir)
|
||||
submission = submission_state(skill_dir, task, submissions_dir)
|
||||
source_accepted = bool(state.get("accepted"))
|
||||
accepted = source_accepted and submission.get("status") == "submitted"
|
||||
return {
|
||||
"key": task["key"],
|
||||
"label": task["label"],
|
||||
@@ -165,6 +138,7 @@ def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) ->
|
||||
"owner": task["owner"],
|
||||
"status": "accepted" if accepted else "pending",
|
||||
"source_status": task["status"],
|
||||
"source_accepted": source_accepted,
|
||||
"current": task["current"],
|
||||
"objective": task["objective"],
|
||||
"provenance_requirements": PROVENANCE_REQUIREMENTS.get(task["key"], ["release reviewer evidence"]),
|
||||
@@ -186,7 +160,9 @@ def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) ->
|
||||
def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
|
||||
plan = build_plan(skill_dir, generated_at)
|
||||
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
|
||||
entries = [build_entry(skill_dir, task, submissions_dir) for task in plan["tasks"]]
|
||||
evidence_requirements = plan.get("evidence_requirements") or plan.get("tasks", [])
|
||||
entries = [build_entry(skill_dir, task, submissions_dir) for task in evidence_requirements]
|
||||
source_accepted_count = sum(1 for entry in entries if entry.get("source_accepted") is True)
|
||||
accepted_count = sum(1 for entry in entries if entry["status"] == "accepted")
|
||||
pending_count = len(entries) - accepted_count
|
||||
human_pending_count = sum(1 for entry in entries if entry["category"] == "human" and entry["status"] == "pending")
|
||||
@@ -197,7 +173,12 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
|
||||
submitted_but_pending_count = sum(
|
||||
1 for entry in entries if entry["submission_state"]["status"] == "submitted" and entry["status"] == "pending"
|
||||
)
|
||||
ready = pending_count == 0 and plan["summary"].get("world_class_ready") is True
|
||||
source_accepted_without_valid_submission_count = sum(
|
||||
1
|
||||
for entry in entries
|
||||
if entry.get("source_accepted") is True and entry["submission_state"]["status"] != "submitted"
|
||||
)
|
||||
ready = bool(entries) and pending_count == 0 and plan["summary"].get("world_class_ready") is True
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": True,
|
||||
@@ -205,6 +186,7 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": {
|
||||
"ledger_entry_count": len(entries),
|
||||
"source_accepted_count": source_accepted_count,
|
||||
"accepted_count": accepted_count,
|
||||
"pending_count": pending_count,
|
||||
"human_pending_count": human_pending_count,
|
||||
@@ -213,6 +195,7 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
|
||||
"missing_submission_count": missing_submission_count,
|
||||
"invalid_submission_count": invalid_submission_count,
|
||||
"submitted_but_pending_count": submitted_but_pending_count,
|
||||
"source_accepted_without_valid_submission_count": source_accepted_without_valid_submission_count,
|
||||
"overclaim_guard_active": True,
|
||||
"ready_to_claim_world_class": ready,
|
||||
"decision": "ready-for-completion-audit" if ready else "evidence-pending",
|
||||
@@ -222,10 +205,12 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
|
||||
"json": "reports/world_class_evidence_plan.json",
|
||||
"markdown": "reports/world_class_evidence_plan.md",
|
||||
"task_count": plan["summary"].get("task_count", 0),
|
||||
"evidence_requirement_count": len(evidence_requirements),
|
||||
},
|
||||
"submissions": {
|
||||
"directory": rel_path(submissions_dir, skill_dir),
|
||||
"ledger_counts_submission_as_completion": False,
|
||||
"source_pass_requires_valid_submission": True,
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_evidence_ledger.json",
|
||||
@@ -247,16 +232,18 @@ def render_markdown(ledger: dict[str, Any]) -> str:
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
|
||||
f"- entries: `{summary['ledger_entry_count']}`",
|
||||
f"- source accepted: `{summary.get('source_accepted_count', 0)}`",
|
||||
f"- accepted: `{summary['accepted_count']}`",
|
||||
f"- pending: `{summary['pending_count']}`",
|
||||
f"- human pending: `{summary['human_pending_count']}`",
|
||||
f"- external pending: `{summary['external_pending_count']}`",
|
||||
f"- submitted entries: `{summary['submitted_entry_count']}`",
|
||||
f"- submitted but pending: `{summary['submitted_but_pending_count']}`",
|
||||
f"- source accepted without valid submission: `{summary.get('source_accepted_without_valid_submission_count', 0)}`",
|
||||
f"- invalid submissions: `{summary['invalid_submission_count']}`",
|
||||
f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`",
|
||||
"",
|
||||
"This ledger records the current evidence state. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.",
|
||||
"This ledger records the current evidence state. It requires both passing source evidence and a validated intake submission with artifact SHA-256 checks before accepting an item. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.",
|
||||
"",
|
||||
"## Ledger",
|
||||
"",
|
||||
|
||||
@@ -175,11 +175,14 @@ def build_task(item: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
def build_plan(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
audit = build_audit(skill_dir, generated_at)
|
||||
tasks = [build_task(item) for item in audit["items"] if item["status"] != "pass"]
|
||||
evidence_keys = set(TASK_TEMPLATES)
|
||||
evidence_requirements = [build_task(item) for item in audit["items"] if item["key"] in evidence_keys]
|
||||
tasks = [task for task in evidence_requirements if task["status"] != "pass"]
|
||||
tasks.extend(build_task(item) for item in audit["items"] if item["status"] != "pass" and item["key"] not in evidence_keys)
|
||||
category_counts: dict[str, int] = {}
|
||||
for task in tasks:
|
||||
category_counts[task["category"]] = category_counts.get(task["category"], 0) + 1
|
||||
ready = len(tasks) == 0 and audit["summary"].get("world_class_ready") is True
|
||||
audit_world_class_ready = audit["summary"].get("world_class_ready") is True
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": audit["ok"],
|
||||
@@ -188,14 +191,18 @@ def build_plan(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
"summary": {
|
||||
"audit_decision": audit["summary"]["decision"],
|
||||
"world_class_ready": bool(audit["summary"]["world_class_ready"]),
|
||||
"ready_to_claim_world_class": ready,
|
||||
"audit_world_class_ready": audit_world_class_ready,
|
||||
"ready_to_claim_world_class": False,
|
||||
"ledger_completion_required": True,
|
||||
"evidence_requirement_count": len(evidence_requirements),
|
||||
"task_count": len(tasks),
|
||||
"human_task_count": category_counts.get("human", 0),
|
||||
"external_task_count": category_counts.get("external", 0),
|
||||
"review_task_count": category_counts.get("review", 0),
|
||||
"decision": "ready-for-completion-audit" if ready else "collect-external-evidence",
|
||||
"decision": "audit-ready-ledger-required" if audit_world_class_ready and not tasks else "collect-external-evidence",
|
||||
},
|
||||
"tasks": tasks,
|
||||
"evidence_requirements": evidence_requirements,
|
||||
"source_audit": {
|
||||
"json": "reports/skill_os2_audit.json",
|
||||
"markdown": "reports/skill_os2_audit.md",
|
||||
@@ -222,11 +229,13 @@ def render_markdown(plan: dict[str, Any]) -> str:
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- audit decision: `{summary['audit_decision']}`",
|
||||
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
|
||||
f"- ledger completion required: `{str(summary.get('ledger_completion_required', True)).lower()}`",
|
||||
f"- evidence requirements: `{summary.get('evidence_requirement_count', 0)}`",
|
||||
f"- tasks: `{summary['task_count']}`",
|
||||
f"- human tasks: `{summary['human_task_count']}`",
|
||||
f"- external tasks: `{summary['external_task_count']}`",
|
||||
"",
|
||||
"This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan as completion.",
|
||||
"This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan or source-report pass as completion; the ledger must still validate accepted submissions.",
|
||||
"",
|
||||
"## Task Table",
|
||||
"",
|
||||
@@ -239,7 +248,7 @@ def render_markdown(plan: dict[str, Any]) -> str:
|
||||
f"| `{task['key']}` | `{task['status']}` | `{task['category']}` | {task['owner']} | {current} |"
|
||||
)
|
||||
if not plan["tasks"]:
|
||||
lines.append("| `none` | `pass` | `none` | none | all evidence collected |")
|
||||
lines.append("| `none` | `pass` | `none` | none | audit gaps closed; ledger validation still required |")
|
||||
for task in plan["tasks"]:
|
||||
lines.extend(
|
||||
[
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to share intake validation and artifact integrity checks."
|
||||
|
||||
REQUIRED_TOP_LEVEL = [
|
||||
"schema_version",
|
||||
"evidence_key",
|
||||
"template_only",
|
||||
"category",
|
||||
"source_type",
|
||||
"submitted_by",
|
||||
"submitted_at",
|
||||
"summary",
|
||||
"artifact_refs",
|
||||
"provenance",
|
||||
"privacy",
|
||||
"anti_overclaim",
|
||||
"attestation",
|
||||
]
|
||||
REQUIRED_PRIVACY_FALSE = [
|
||||
"raw_user_content_included",
|
||||
"raw_provider_prompt_included",
|
||||
"credentials_included",
|
||||
"secrets_included",
|
||||
]
|
||||
REQUIRED_ANTI_OVERCLAIM_FALSE = [
|
||||
"planned_work_counts_as_evidence",
|
||||
"metadata_fallback_counts_as_native_enforcement",
|
||||
"pending_review_counts_as_human_decision",
|
||||
"local_command_runner_counts_as_provider_model",
|
||||
]
|
||||
REQUIRED_ATTESTATION_TRUE = [
|
||||
"real_external_or_human_evidence",
|
||||
"reviewer_or_operator_identity_present",
|
||||
"artifact_refs_reviewed",
|
||||
"privacy_contract_satisfied",
|
||||
]
|
||||
EXPECTED_SOURCE_TYPES = {
|
||||
"provider-holdout": "provider-output-eval",
|
||||
"human-adjudication": "blind-ab-review",
|
||||
"native-permission-enforcement": "runtime-permission-guard",
|
||||
"native-client-telemetry": "native-client-telemetry",
|
||||
}
|
||||
SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
|
||||
DISALLOWED_REAL_ARTIFACTS = {"reports/telemetry_events.jsonl"}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_json_with_status(path: Path) -> tuple[dict[str, Any], str]:
|
||||
if not path.exists():
|
||||
return {}, "missing"
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}, "invalid-json"
|
||||
if not isinstance(payload, dict):
|
||||
return {}, "invalid-json"
|
||||
return payload, "present"
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def add_error(errors: list[str], condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
errors.append(message)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def resolve_artifact_path(raw_path: str, root: Path) -> tuple[Path | None, str | None]:
|
||||
text = raw_path.strip()
|
||||
if any(token in text for token in ("<", ">", "*", "?")):
|
||||
return None, "artifact_refs path must be concrete, not a placeholder or glob"
|
||||
candidate = Path(text)
|
||||
if candidate.is_absolute():
|
||||
return None, "artifact_refs path must be relative to the skill directory"
|
||||
if ".." in candidate.parts:
|
||||
return None, "artifact_refs path must not escape the skill directory"
|
||||
resolved = (root / candidate).resolve()
|
||||
try:
|
||||
resolved.relative_to(root.resolve())
|
||||
except ValueError:
|
||||
return None, "artifact_refs path must stay inside the skill directory"
|
||||
return resolved, None
|
||||
|
||||
|
||||
def validate_artifact_refs(
|
||||
payload: dict[str, Any],
|
||||
errors: list[str],
|
||||
*,
|
||||
root: Path,
|
||||
template_expected: bool,
|
||||
) -> dict[str, int]:
|
||||
refs = payload.get("artifact_refs")
|
||||
add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference")
|
||||
stats = {
|
||||
"artifact_ref_count": len(refs) if isinstance(refs, list) else 0,
|
||||
"artifact_existing_count": 0,
|
||||
"artifact_sha256_verified_count": 0,
|
||||
}
|
||||
if not isinstance(refs, list):
|
||||
return stats
|
||||
for index, ref in enumerate(refs):
|
||||
if not isinstance(ref, dict):
|
||||
errors.append(f"artifact_refs[{index}] must be an object")
|
||||
continue
|
||||
path_text = str(ref.get("path", "")).strip()
|
||||
add_error(errors, bool(path_text), f"artifact_refs[{index}].path is required")
|
||||
add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required")
|
||||
add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content")
|
||||
if template_expected or not path_text:
|
||||
continue
|
||||
resolved, path_error = resolve_artifact_path(path_text, root)
|
||||
if path_error:
|
||||
errors.append(f"artifact_refs[{index}].path {path_error}")
|
||||
continue
|
||||
rel = rel_path(resolved, root)
|
||||
if rel in DISALLOWED_REAL_ARTIFACTS:
|
||||
errors.append(f"artifact_refs[{index}].path must not reference raw local telemetry logs")
|
||||
if not resolved.exists() or not resolved.is_file():
|
||||
errors.append(f"artifact_refs[{index}].path does not exist as a local file")
|
||||
continue
|
||||
stats["artifact_existing_count"] += 1
|
||||
declared = str(ref.get("sha256", "")).strip()
|
||||
if not declared:
|
||||
errors.append(f"artifact_refs[{index}].sha256 is required for a real submission")
|
||||
continue
|
||||
if not SHA256_RE.match(declared):
|
||||
errors.append(f"artifact_refs[{index}].sha256 must be a 64-character hex digest")
|
||||
continue
|
||||
actual = sha256_file(resolved)
|
||||
if actual.lower() != declared.lower():
|
||||
errors.append(f"artifact_refs[{index}].sha256 does not match local artifact")
|
||||
continue
|
||||
stats["artifact_sha256_verified_count"] += 1
|
||||
return stats
|
||||
|
||||
|
||||
def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None:
|
||||
key = str(payload.get("evidence_key", ""))
|
||||
provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
|
||||
if key == "provider-holdout":
|
||||
add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required")
|
||||
add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required")
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("credential_material_committed") is False,
|
||||
"provider-holdout must attest credential_material_committed is false",
|
||||
)
|
||||
elif key == "human-adjudication":
|
||||
add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required")
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("answer_key_opened_after_decisions") is True,
|
||||
"human-adjudication must attest answer_key_opened_after_decisions is true",
|
||||
)
|
||||
elif key == "native-permission-enforcement":
|
||||
add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required")
|
||||
add_error(
|
||||
errors,
|
||||
bool(str(provenance.get("guard_location", "")).strip()),
|
||||
"native-permission-enforcement provenance.guard_location is required",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
str(provenance.get("guard_scope", "")).strip()
|
||||
in {"target-client-native", "external-installer-runtime-guard"},
|
||||
"native-permission-enforcement provenance.guard_scope must be target-client-native or external-installer-runtime-guard",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("guard_blocks_undeclared_capability") is True,
|
||||
"native-permission-enforcement must attest guard_blocks_undeclared_capability is true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("metadata_fallback_retained_for_other_targets") is True,
|
||||
"native-permission-enforcement must retain metadata fallback for non-native targets",
|
||||
)
|
||||
elif key == "native-client-telemetry":
|
||||
add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required")
|
||||
add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external")
|
||||
add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only")
|
||||
|
||||
|
||||
def validate_payload(
|
||||
payload: dict[str, Any],
|
||||
entry: dict[str, Any],
|
||||
*,
|
||||
path: Path,
|
||||
root: Path,
|
||||
template_expected: bool,
|
||||
) -> dict[str, Any]:
|
||||
errors: list[str] = []
|
||||
for key in REQUIRED_TOP_LEVEL:
|
||||
add_error(errors, key in payload, f"missing {key}")
|
||||
evidence_key = str(entry.get("key", ""))
|
||||
add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0")
|
||||
add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}")
|
||||
add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}")
|
||||
add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}")
|
||||
add_error(
|
||||
errors,
|
||||
payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key),
|
||||
f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}",
|
||||
)
|
||||
add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required")
|
||||
add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required")
|
||||
add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required")
|
||||
artifact_integrity = validate_artifact_refs(payload, errors, root=root, template_expected=template_expected)
|
||||
privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {}
|
||||
for key in REQUIRED_PRIVACY_FALSE:
|
||||
add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false")
|
||||
anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {}
|
||||
for key in REQUIRED_ANTI_OVERCLAIM_FALSE:
|
||||
add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false")
|
||||
validate_evidence_specific(payload, errors)
|
||||
attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {}
|
||||
if not template_expected:
|
||||
for key in REQUIRED_ATTESTATION_TRUE:
|
||||
add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission")
|
||||
return {
|
||||
"path": rel_path(path, root),
|
||||
"evidence_key": evidence_key,
|
||||
"status": "pass" if not errors else "fail",
|
||||
"template_only": template_expected,
|
||||
"artifact_integrity": artifact_integrity,
|
||||
"errors": errors,
|
||||
}
|
||||
@@ -107,6 +107,7 @@
|
||||
"scripts/upgrade_check.py",
|
||||
"scripts/validate_skill.py",
|
||||
"scripts/verify_package.py",
|
||||
"scripts/world_class_evidence_contract.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_create_commands.py",
|
||||
|
||||
@@ -95,6 +95,7 @@ def main() -> None:
|
||||
"scripts/skill_report_layout.py",
|
||||
"scripts/skill_report_metrics.py",
|
||||
"scripts/skill_report_model.py",
|
||||
"scripts/world_class_evidence_contract.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_parser.py",
|
||||
"scripts/yao_cli_telemetry.py",
|
||||
@@ -109,6 +110,7 @@ def main() -> None:
|
||||
assert "skill_report_layout.py" not in warning_text, payload["warnings"]
|
||||
assert "skill_report_metrics.py" not in warning_text, payload["warnings"]
|
||||
assert "skill_report_model.py" not in warning_text, payload["warnings"]
|
||||
assert "world_class_evidence_contract.py" not in warning_text, payload["warnings"]
|
||||
assert "yao_cli_config.py" not in warning_text, payload["warnings"]
|
||||
assert "yao_cli_parser.py" not in warning_text, payload["warnings"]
|
||||
assert "yao_cli_telemetry.py" not in warning_text, payload["warnings"]
|
||||
|
||||
@@ -52,13 +52,23 @@ def main() -> None:
|
||||
|
||||
safe_surface = TMP / "safe.md"
|
||||
safe_surface.write_text("ready to claim world-class: `false`\nworld-class evidence is pending.\n", encoding="utf-8")
|
||||
safe_proc = run_guard("--claim-surface", str(safe_surface), check=True)
|
||||
safe_proc = run_guard(
|
||||
"--claim-surface", str(safe_surface),
|
||||
"--output-json", str(TMP / "safe_guard.json"),
|
||||
"--output-md", str(TMP / "safe_guard.md"),
|
||||
check=True,
|
||||
)
|
||||
safe_payload = json.loads(safe_proc.stdout)
|
||||
assert safe_payload["summary"]["violation_count"] == 0, safe_payload
|
||||
|
||||
unsafe_surface = TMP / "unsafe.md"
|
||||
unsafe_surface.write_text("ready to claim world-class: `true`\n世界级已完成\n", encoding="utf-8")
|
||||
unsafe_proc = run_guard("--claim-surface", str(unsafe_surface), check=False)
|
||||
unsafe_proc = run_guard(
|
||||
"--claim-surface", str(unsafe_surface),
|
||||
"--output-json", str(TMP / "unsafe_guard.json"),
|
||||
"--output-md", str(TMP / "unsafe_guard.md"),
|
||||
check=False,
|
||||
)
|
||||
assert unsafe_proc.returncode == 2, unsafe_proc.stdout
|
||||
unsafe_payload = json.loads(unsafe_proc.stdout)
|
||||
assert unsafe_payload["ok"] is False, unsafe_payload
|
||||
@@ -72,6 +82,10 @@ def main() -> None:
|
||||
relative_proc = run_guard(
|
||||
"--claim-surface",
|
||||
relative_unsafe_surface.relative_to(ROOT).as_posix(),
|
||||
"--output-json",
|
||||
str(TMP / "relative_unsafe_guard.json"),
|
||||
"--output-md",
|
||||
str(TMP / "relative_unsafe_guard.md"),
|
||||
check=False,
|
||||
cwd=TMP,
|
||||
)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -11,7 +12,15 @@ SCRIPT = ROOT / "scripts" / "render_world_class_evidence_ledger.py"
|
||||
TMP = ROOT / "tests" / "tmp_world_class_evidence_ledger"
|
||||
|
||||
|
||||
def provider_submission() -> dict:
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def provider_submission(artifact_root: Path = ROOT) -> dict:
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"evidence_key": "provider-holdout",
|
||||
@@ -26,8 +35,26 @@ def provider_submission() -> dict:
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"kind": "aggregate-report",
|
||||
"contains_raw_content": False,
|
||||
"sha256": sha256_file(artifact_root / "reports" / "output_execution_runs.json"),
|
||||
}
|
||||
],
|
||||
"provenance": {
|
||||
"provider": "openai",
|
||||
"model": "gpt-4.1-mini",
|
||||
"credential_material_committed": False,
|
||||
},
|
||||
"privacy": {
|
||||
"raw_user_content_included": False,
|
||||
"raw_provider_prompt_included": False,
|
||||
"credentials_included": False,
|
||||
"secrets_included": False,
|
||||
},
|
||||
"anti_overclaim": {
|
||||
"planned_work_counts_as_evidence": False,
|
||||
"metadata_fallback_counts_as_native_enforcement": False,
|
||||
"pending_review_counts_as_human_decision": False,
|
||||
"local_command_runner_counts_as_provider_model": False,
|
||||
},
|
||||
"attestation": {
|
||||
"real_external_or_human_evidence": True,
|
||||
"reviewer_or_operator_identity_present": True,
|
||||
@@ -136,8 +163,90 @@ def main() -> None:
|
||||
submitted_provider = {entry["key"]: entry for entry in submitted_payload["entries"]}["provider-holdout"]
|
||||
assert submitted_provider["status"] == "pending", submitted_provider
|
||||
assert submitted_provider["submission_state"]["status"] == "submitted", submitted_provider
|
||||
assert submitted_provider["submission_state"]["artifact_sha256_verified_count"] == 1, submitted_provider
|
||||
assert submitted_provider["submission_state"]["attested_real_evidence"] is True, submitted_provider
|
||||
assert submitted_provider["submission_state"]["ledger_counts_as_completion"] is False, submitted_provider
|
||||
|
||||
accepted_source_skill = TMP / "accepted_source_skill"
|
||||
(accepted_source_skill / "reports").mkdir(parents=True)
|
||||
(accepted_source_skill / "reports" / "output_execution_runs.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"summary": {
|
||||
"model_executed_count": 1,
|
||||
"timing_observed_count": 1,
|
||||
"token_observed_count": 1,
|
||||
}
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
accepted_source_proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
str(accepted_source_skill),
|
||||
"--output-json",
|
||||
str(TMP / "accepted_source_without_submission.json"),
|
||||
"--output-md",
|
||||
str(TMP / "accepted_source_without_submission.md"),
|
||||
"--generated-at",
|
||||
"2026-06-13",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
accepted_source_payload = json.loads(accepted_source_proc.stdout)
|
||||
accepted_source_summary = accepted_source_payload["summary"]
|
||||
assert accepted_source_summary["source_accepted_count"] == 1, accepted_source_summary
|
||||
assert accepted_source_summary["accepted_count"] == 0, accepted_source_summary
|
||||
assert accepted_source_summary["source_accepted_without_valid_submission_count"] == 1, accepted_source_summary
|
||||
accepted_source_provider = {
|
||||
entry["key"]: entry for entry in accepted_source_payload["entries"]
|
||||
}["provider-holdout"]
|
||||
assert accepted_source_provider["source_accepted"] is True, accepted_source_provider
|
||||
assert accepted_source_provider["status"] == "pending", accepted_source_provider
|
||||
assert accepted_source_provider["submission_state"]["status"] == "missing", accepted_source_provider
|
||||
|
||||
accepted_submissions = TMP / "accepted_submissions"
|
||||
accepted_submissions.mkdir()
|
||||
(accepted_submissions / "provider-holdout.json").write_text(
|
||||
json.dumps(provider_submission(accepted_source_skill), ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
accepted_proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
str(accepted_source_skill),
|
||||
"--output-json",
|
||||
str(TMP / "accepted_provider_ledger.json"),
|
||||
"--output-md",
|
||||
str(TMP / "accepted_provider_ledger.md"),
|
||||
"--submissions-dir",
|
||||
str(accepted_submissions),
|
||||
"--generated-at",
|
||||
"2026-06-13",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
accepted_payload = json.loads(accepted_proc.stdout)
|
||||
accepted_summary = accepted_payload["summary"]
|
||||
assert accepted_summary["source_accepted_count"] == 1, accepted_summary
|
||||
assert accepted_summary["accepted_count"] == 1, accepted_summary
|
||||
assert accepted_summary["pending_count"] == 3, accepted_summary
|
||||
assert accepted_summary["source_accepted_without_valid_submission_count"] == 0, accepted_summary
|
||||
accepted_provider = {entry["key"]: entry for entry in accepted_payload["entries"]}["provider-holdout"]
|
||||
assert accepted_provider["status"] == "accepted", accepted_provider
|
||||
assert accepted_provider["submission_state"]["status"] == "submitted", accepted_provider
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
|
||||
@@ -38,18 +38,22 @@ def main() -> None:
|
||||
assert payload["ok"] is True, payload
|
||||
assert payload["summary"]["decision"] == "collect-external-evidence", payload
|
||||
assert payload["summary"]["ready_to_claim_world_class"] is False, payload
|
||||
assert payload["summary"]["ledger_completion_required"] is True, payload
|
||||
assert payload["summary"]["evidence_requirement_count"] == 4, payload
|
||||
assert payload["summary"]["task_count"] == 4, payload
|
||||
assert payload["summary"]["human_task_count"] == 1, payload
|
||||
assert payload["summary"]["external_task_count"] == 3, payload
|
||||
assert payload["artifacts"]["ledger"] == "reports/world_class_evidence_ledger.md", payload
|
||||
assert payload["artifacts"]["intake"] == "reports/world_class_evidence_intake.md", payload
|
||||
tasks = {item["key"]: item for item in payload["tasks"]}
|
||||
requirements = {item["key"]: item for item in payload["evidence_requirements"]}
|
||||
assert set(tasks) == {
|
||||
"provider-holdout",
|
||||
"human-adjudication",
|
||||
"native-permission-enforcement",
|
||||
"native-client-telemetry",
|
||||
}, tasks
|
||||
assert set(requirements) == set(tasks), requirements
|
||||
assert any("--provider-runner openai" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"]
|
||||
assert any("world-class-intake" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"]
|
||||
assert any("evidence/world_class/templates/provider-holdout.intake.json" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"]
|
||||
@@ -68,6 +72,7 @@ def main() -> None:
|
||||
assert "World-Class Evidence Plan" in markdown, markdown
|
||||
assert "`provider-holdout`" in markdown, markdown
|
||||
assert "ready to claim world-class: `false`" in markdown, markdown
|
||||
assert "ledger completion required: `true`" in markdown, markdown
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user