diff --git a/AGENTS.md b/AGENTS.md index 6531880..586f8b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,11 @@ python3 scripts/registry_audit.py . --generated-at "$GENERATED_AT" python3 scripts/verify_package.py . --package-dir dist --expectations evals/packaging_expectations.json --registry-json reports/registry_audit.json --output-json reports/package_verification.json --output-md reports/package_verification.md --require-zip --generated-at "$GENERATED_AT" python3 scripts/registry_audit.py . --generated-at "$GENERATED_AT" python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json --current-package-json reports/registry_audit.json --output-json reports/upgrade_check.json --output-md reports/upgrade_check.md --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_evidence_plan.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_evidence_ledger.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_evidence_intake.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_submission_review.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_overview.py . python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json ``` @@ -69,6 +74,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept - `scripts/render_skill_os2_audit.py`: requirement-by-requirement Skill OS 2.0 completion audit. Keep local evidence, human-required gaps, and external-required gaps separate so reports do not overclaim world-class readiness. - `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete. - `scripts/render_world_class_evidence_ledger.py`: machine-checkable acceptance ledger for the remaining world-class evidence gaps. Keep anti-overclaim guards explicit so planned work, metadata fallbacks, pending review, and local command runners never count as final evidence. +- `scripts/render_world_class_submission_review.py`: read-only queue for external and human evidence packets after intake validation. Keep it from accepting evidence; it may only compare packet validity, source evidence checks, and ledger state. - `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable. - `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation. - `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers. diff --git a/Makefile b/Makefile index add83b7..be0d84c 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -106,6 +106,9 @@ world-class-ledger-check: world-class-intake-check: $(PYTHON) tests/verify_world_class_evidence_intake.py +world-class-submission-review-check: + $(PYTHON) tests/verify_world_class_submission_review.py + world-class-claim-guard-check: $(PYTHON) tests/verify_world_class_claim_guard.py @@ -226,11 +229,11 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 5951cfb..50bda2c 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ python3 scripts/yao.py world-class-evidence . python3 scripts/yao.py world-class-submission-kit . --output-dir /tmp/yao-world-class-submission-kit python3 scripts/yao.py world-class-ledger . python3 scripts/yao.py world-class-intake . +python3 scripts/yao.py world-class-submission-review . python3 scripts/yao.py world-class-claim-guard . python3 scripts/yao.py benchmark-reproducibility . ``` @@ -296,6 +297,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - remaining world-class evidence tasks are published in [reports/world_class_evidence_plan.md](reports/world_class_evidence_plan.md) - current world-class evidence acceptance state is published in [reports/world_class_evidence_ledger.md](reports/world_class_evidence_ledger.md) - world-class evidence intake readiness is published in [reports/world_class_evidence_intake.md](reports/world_class_evidence_intake.md) +- world-class submission review queue is published in [reports/world_class_submission_review.md](reports/world_class_submission_review.md) - world-class public claim guard status is published in [reports/world_class_claim_guard.md](reports/world_class_claim_guard.md) - benchmark reproducibility evidence is published in [reports/benchmark_reproducibility.md](reports/benchmark_reproducibility.md) - target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md) @@ -414,7 +416,7 @@ Utility scripts that make the meta-skill operational: - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs - `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence -- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, python-compat, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, world-class-claim-guard, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint +- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, python-compat, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, world-class-submission-kit, world-class-submission-review, world-class-claim-guard, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot - `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger @@ -428,6 +430,7 @@ Utility scripts that make the meta-skill operational: - `render_world_class_evidence_plan.py`: renders executable evidence tasks for remaining world-class gaps without treating planned external work as completed evidence - `render_world_class_evidence_ledger.py`: renders a machine-checkable ledger for current world-class evidence acceptance, anti-overclaim guards, provenance requirements, and privacy contracts - `render_world_class_evidence_intake.py`: validates world-class external and human evidence packets against provenance, privacy, artifact, and anti-overclaim requirements before ledger review +- `render_world_class_submission_review.py`: renders a read-only queue that compares submissions, intake validation, source evidence, and ledger state without accepting evidence - `render_world_class_claim_guard.py`: scans README, docs, and reports for premature world-class completion claims while accepted evidence is still pending - `render_benchmark_reproducibility.py`: renders methodology, artifact, failure-disclosure, and reproduction-command evidence for public benchmark claims - `python_compat_check.py`: checks Python source for supported-runtime compatibility hazards such as Python 3.11 f-string expression backslashes diff --git a/evidence/world_class/README.md b/evidence/world_class/README.md index 642a279..80859d0 100644 --- a/evidence/world_class/README.md +++ b/evidence/world_class/README.md @@ -9,6 +9,7 @@ Run: ```bash python3 scripts/yao.py world-class-submission-kit . --output-dir /tmp/yao-world-class-submission-kit python3 scripts/yao.py world-class-intake . +python3 scripts/yao.py world-class-submission-review . ``` The intake validator checks: @@ -23,4 +24,6 @@ The generated intake report also includes an `operator_checklist` for each pendi The submission kit command creates editable JSON drafts plus a local README for an external operator or human reviewer. Those drafts keep `template_only: true` and do not count as evidence until the real run or review exists, the packet is edited truthfully, and `world-class-intake` validates it. +The submission review command renders a read-only queue that compares valid packets with the source evidence checks and current ledger state. It is for reviewer triage only; it does not accept evidence or make the world-class claim true. + Accepted intake means "ready for ledger review", not "world-class complete". The ledger remains the source of truth for `ready_to_claim_world_class`. diff --git a/registry/index.json b/registry/index.json index ce5e897..5205ba2 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "4871c36233a53615c8e19c1c59449af60e732fff2d389742ca510422d7cbac75" + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 32f8670..d475731 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4871c36233a53615c8e19c1c59449af60e732fff2d389742ca510422d7cbac75", - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 125e304..de4a7b4 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T22:21:26Z", + "generated_at": "2026-06-13T22:37:14Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index ca779cc..a37a50b 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,15 +4,15 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 147, - "script_file_count": 91, - "test_file_count": 56, + "python_file_count": 149, + "script_file_count": 92, + "test_file_count": 57, "internal_module_count": 19, - "cli_script_count": 73, + "cli_script_count": 74, "command_handler_count": 33, "warn_line_threshold": 900, "block_line_threshold": 1500, - "largest_file_lines": 899, + "largest_file_lines": 898, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" @@ -20,21 +20,21 @@ "largest_files": [ { "path": "scripts/render_review_studio.py", - "lines": 899, + "lines": 898, "kind": "cli-script", "severity": "pass", "recommendation": "Move data loading and large section renderers into focused review_studio_* modules." }, { "path": "tests/verify_yao_cli.py", - "lines": 871, + "lines": 891, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { "path": "scripts/yao.py", - "lines": 840, + "lines": 841, "kind": "cli-script", "severity": "pass", "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." @@ -69,7 +69,7 @@ }, { "path": "scripts/yao_cli_parser.py", - "lines": 703, + "lines": 714, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 9b9540e..861ca2f 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,13 +5,13 @@ Generated at: `2026-06-13` ## Summary - decision: `pass` -- python files: `147` -- scripts: `91` -- tests: `56` +- python files: `149` +- scripts: `92` +- tests: `57` - internal modules: `19` -- CLI scripts: `73` +- CLI scripts: `74` - Yao CLI command handlers: `33` -- largest file lines: `899` +- largest file lines: `898` - hotspots: `0` - blockers: `0` @@ -25,14 +25,14 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | -| `scripts/render_review_studio.py` | `899` | `cli-script` | `pass` | -| `tests/verify_yao_cli.py` | `871` | `test` | `pass` | -| `scripts/yao.py` | `840` | `cli-script` | `pass` | +| `scripts/render_review_studio.py` | `898` | `cli-script` | `pass` | +| `tests/verify_yao_cli.py` | `891` | `test` | `pass` | +| `scripts/yao.py` | `841` | `cli-script` | `pass` | | `scripts/skill_report_model.py` | `782` | `internal-module` | `pass` | | `scripts/compile_skill.py` | `734` | `cli-script` | `pass` | | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | -| `scripts/yao_cli_parser.py` | `703` | `internal-module` | `pass` | +| `scripts/yao_cli_parser.py` | `714` | `internal-module` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `674` | `cli-script` | `pass` | | `scripts/skill_report_layout.py` | `653` | `internal-module` | `pass` | diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 02ae36b..46b6115 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,24 +3,24 @@ "ok": true, "generated_at": "2026-06-14", "skill_dir": ".", - "commit": "b540f580b373cb9c0f575b687fb82c5a16748a5d", + "commit": "30539bed7a29dd88c2f289dd0c115da9834dd5d9", "git_status": { "available": true, "dirty": true, - "changed_file_count": 48, + "changed_file_count": 60, "sample": [ - " M .gitignore", + " M AGENTS.md", + " M Makefile", " M README.md", " M evidence/world_class/README.md", " M registry/index.json", " M registry/packages/yao-meta-skill.json", " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", " M reports/architecture_maintainability.json", " M reports/architecture_maintainability.md", " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", - " M reports/compiled_targets.json", - " M reports/context_budget.json" + " M reports/benchmark_reproducibility.md" ] }, "summary": { @@ -42,7 +42,7 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 48 + "changed_file_count": 60 }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -115,8 +115,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7966, - "sha256": "9456b0729b322d3f9fdf6f0929c0010d649df94eccc9597f4e0f67115dc8ea1b" + "bytes": 7967, + "sha256": "34fa0a084b7537cca412b4bbea4018639ceac9e98ddf83a6d25ee2dc7f7dbe4f" }, { "label": "blind_review", @@ -150,43 +150,43 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 93677, - "sha256": "2aa9f4b0a7e2ca36d7f8e87e76bbc224bb23f9962f86d2c0f2b3cefd7deca221" + "bytes": 94809, + "sha256": "1d256935f965a265ae5c60e8600a081db8aebe1fdeff7068c5d08fb521fe653b" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 19564, - "sha256": "bbab01ebaa5b6986b0a3bfcf9825b15efdea5b98377a126fb0708e9fa67d2acb" + "bytes": 19840, + "sha256": "9b8f5b17fe2711b17557366edda49a89bbf2e378576de52d7de798137dc8a421" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "69f9f9f4663e291275c0c50caa86aac68d2579f9e02abf3f6dc9e03b64a77557" + "sha256": "c3ae3224b9339aec1e01e695835aa14113bf23979100eda2099fa9609e841c17" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19325, - "sha256": "c8ab851a6f715e866a7b5fca04de075692027d77bc81d4d68e425c5d2dd2a7de" + "sha256": "89dbb682f4a498a1a57312d0bf59527b8cf2713f7b78d065e9f4da05b4104a93" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, "bytes": 8758, - "sha256": "a88a496f58a591e3f81f552347de64e56dd9269bb775121e6bf0fafab795707f" + "sha256": "465f407ccc808169f5380550161f481ab8f0fbc20ef9ab170814fb703b05fe3b" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 13946, - "sha256": "91945bad2dcd8f6a165b1c896932a61ee64f43bfc9cca2091f3a921ef7817c0c" + "sha256": "340c10df7350d74cbfe3fd77461ce2cbab201198fd93fd276e7273eb6f6222e7" }, { "label": "world_class_evidence_plan", @@ -213,8 +213,8 @@ "label": "world_class_claim_guard", "path": "reports/world_class_claim_guard.json", "exists": true, - "bytes": 7995, - "sha256": "608c954d1af917a0e062d90d976a157a44a578e1137e3759209be90c100fecc2" + "bytes": 8093, + "sha256": "24396b9ecb4351cb9e3dff3d69b83bb940bf1507387a086cb0a08ca8cfdfa618" } ], "missing_artifacts": [], diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 76ac717..55c5333 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,7 +1,7 @@ # Benchmark Reproducibility Generated at: `2026-06-14` -Commit: `b540f580b373cb9c0f575b687fb82c5a16748a5d` +Commit: `30539bed7a29dd88c2f289dd0c115da9834dd5d9` Working tree dirty at generation: `true` ## Summary @@ -16,7 +16,7 @@ Working tree dirty at generation: `true` - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- changed files at generation: `48` +- changed files at generation: `60` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. @@ -40,21 +40,21 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `9456b0729b32` | +| output_execution | `reports/output_execution_runs.json` | present | `34fa0a084b75` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `5765254c4309` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | -| trust_report | `reports/security_trust_report.json` | present | `2aa9f4b0a7e2` | -| python_compatibility | `reports/python_compatibility.json` | present | `bbab01ebaa5b` | -| registry_audit | `reports/registry_audit.json` | present | `69f9f9f4663e` | -| package_verification | `reports/package_verification.json` | present | `c8ab851a6f71` | -| install_simulation | `reports/install_simulation.json` | present | `a88a496f58a5` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `91945bad2dcd` | +| trust_report | `reports/security_trust_report.json` | present | `1d256935f965` | +| python_compatibility | `reports/python_compatibility.json` | present | `9b8f5b17fe27` | +| registry_audit | `reports/registry_audit.json` | present | `c3ae3224b933` | +| package_verification | `reports/package_verification.json` | present | `89dbb682f4a4` | +| install_simulation | `reports/install_simulation.json` | present | `465f407ccc80` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `340c10df7350` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `ed1274b7c18c` | | world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `e6dfdb22dc9f` | | world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `465f8d915a86` | -| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `608c954d1af9` | +| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `24396b9ecb43` | ## Reproduction Commands diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index 3246865..6d5da15 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -204,6 +204,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -223,8 +224,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -383,7 +383,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -436,6 +436,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -484,14 +485,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -513,7 +514,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -626,7 +627,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -679,6 +680,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -727,14 +729,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -754,7 +756,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -1047,6 +1049,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -1066,8 +1069,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -1226,7 +1228,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1279,6 +1281,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -1327,14 +1330,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1356,7 +1359,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -1469,7 +1472,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1522,6 +1525,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -1570,14 +1574,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1597,7 +1601,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -1890,6 +1894,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -1909,8 +1914,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -2069,7 +2073,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2122,6 +2126,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -2170,14 +2175,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2199,7 +2204,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -2305,7 +2310,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2358,6 +2363,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -2406,14 +2412,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2433,7 +2439,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -2717,6 +2723,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -2736,8 +2743,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -2896,7 +2902,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2949,6 +2955,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -2997,14 +3004,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3026,7 +3033,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -3132,7 +3139,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3185,6 +3192,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -3233,14 +3241,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3260,7 +3268,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -3544,6 +3552,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -3563,8 +3572,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -3723,7 +3731,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3776,6 +3784,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -3824,14 +3833,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3853,7 +3862,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -3963,7 +3972,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4016,6 +4025,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -4064,14 +4074,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4091,7 +4101,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, diff --git a/reports/context_budget.json b/reports/context_budget.json index af17c44..8d75430 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1117887, + "other_text_tokens": 1129811, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1118638, - "relevant_file_count": 480, + "estimated_total_text_tokens": 1130562, + "relevant_file_count": 484, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index bed46a8..f289665 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 559, + "archive_entry_count": 563, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 3b13f25..724ac2c 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.47, + "duration_ms": 27.77, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.17, + "duration_ms": 28.86, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.51, + "duration_ms": 28.67, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.11, + "duration_ms": 28.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.82, + "duration_ms": 28.7, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.65, + "duration_ms": 27.96, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.97, + "duration_ms": 28.07, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.12, + "duration_ms": 28.42, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.34, + "duration_ms": 28.64, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.07, + "duration_ms": 28.43, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index f1aee64..298a726 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 32.47 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 33.17 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 32.51 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.11 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 32.82 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 29.65 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.97 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.12 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.34 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.07 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 27.77 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 28.86 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 28.67 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.37 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 28.7 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.96 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.07 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.42 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.64 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.43 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 41a7d3d..97f16cf 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", - "archive_entry_count": 559, + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", + "archive_entry_count": 563, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 4e3d042..cb0c122 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e` +- Archive SHA256: `cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 0f56645..ecfc9c9 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 150, + "file_count": 152, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -394,6 +394,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/render_world_class_submission_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/resource_boundary_check.py", "ok": true, @@ -916,6 +922,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_world_class_submission_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_yao_cli.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index b64a250..f5fbea0 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-13` - decision: `pass` - target python: `3.11` -- files scanned: `150` +- files scanned: `152` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 79cc0a3..5aa97c7 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4871c36233a53615c8e19c1c59449af60e732fff2d389742ca510422d7cbac75", - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "4871c36233a53615c8e19c1c59449af60e732fff2d389742ca510422d7cbac75" + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 7cc916c..0dfa22b 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `4871c36233a53615c8e19c1c59449af60e732fff2d389742ca510422d7cbac75` -- Archive SHA256: `fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e` +- Package SHA256: `68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926` +- Archive SHA256: `cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 1e71e48..1f39e1f 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -434,12 +434,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

91 scripts scanned; secrets found

Py Compat0

150 files scanned for Python 3.11

Arch Debt0

899 largest lines; 33 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

68 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

559 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

92 scripts scanned; secrets found

Py Compat0

152 files scanned for Python 3.11

Arch Debt0

898 largest lines; 33 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

69 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

563 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 91 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 150 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

147 Python files; 0 hotspots; 0 blockers; largest 899 lines; 33 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 92 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 152 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

149 Python files; 0 hotspots; 0 blockers; largest 898 lines; 33 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -449,7 +449,7 @@

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md
验证命令
python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .
@@ -465,7 +465,7 @@

证据路径

- +
@@ -506,12 +506,12 @@
-

信任报告

Secret
0
脚本数
91
网络脚本
3
Help 失败
0
包体哈希
d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac
+

信任报告

Secret
0
脚本数
92
网络脚本
3
Help 失败
0
包体哈希
68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

-

Python 兼容

目标 Python
3.11
文件数
150
问题数
0
语法错误
0
F-string 3.11
0
+

Python 兼容

目标 Python
3.11
文件数
152
问题数
0
语法错误
0
F-string 3.11
0

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线；本地更高版本允许的新语法不能绕过兼容门禁。

@@ -568,18 +568,18 @@
-

声明守卫

台账可声明
否
台账待补
4
声明面
68
违规数
0
Overclaim Guard Active
是
+

声明守卫

台账可声明
否
台账待补
4
声明面
69
违规数
0
Overclaim Guard Active
是

声明边界

claim guard 扫描 README、docs 和 reports 中的完成态表述；ledger 未 ready 时，任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
559
失败数
0
警告数
0
归档哈希
fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
563
失败数
0
警告数
0
归档哈希
cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e
diff --git a/reports/review-studio.json b/reports/review-studio.json index 2d8a527..d6f9362 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 91 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 92 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -67,7 +67,7 @@ "key": "python-compat", "label": "Python 兼容", "status": "pass", - "detail": "Python 3.11; 150 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", + "detail": "Python 3.11; 152 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", "evidence": "reports/python_compatibility.json", "link": "python_compatibility.md" }, @@ -75,7 +75,7 @@ "key": "architecture-maintainability", "label": "架构维护", "status": "pass", - "detail": "147 Python files; 0 hotspots; 0 blockers; largest 899 lines; 33 CLI handlers", + "detail": "149 Python files; 0 hotspots; 0 blockers; largest 898 lines; 33 CLI handlers", "evidence": "reports/architecture_maintainability.json", "link": "architecture_maintainability.md" }, @@ -271,7 +271,7 @@ "priority": "warning", "summary": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。", "why": "世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。", - "source_fix": "reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md", + "source_fix": "reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md", "source_refs": [ { "path": "reports/world_class_evidence_ledger.md", @@ -297,6 +297,14 @@ "exists": true, "link": "world_class_evidence_intake.md" }, + { + "path": "reports/world_class_submission_review.md", + "label": "world-class submission review", + "kind": "report", + "line": 1, + "exists": true, + "link": "world_class_submission_review.md" + }, { "path": "reports/world_class_claim_guard.md", "label": "world-class claim guard", @@ -364,7 +372,7 @@ ], "evidence": "reports/world_class_evidence_ledger.json", "evidence_link": "world_class_evidence_ledger.md", - "verification_command": "python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio ." + "verification_command": "python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio ." } ], "evidence_paths": { @@ -391,6 +399,7 @@ "world_class_evidence_plan": "reports/world_class_evidence_plan.md", "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", "world_class_evidence_intake": "reports/world_class_evidence_intake.md", + "world_class_submission_review": "reports/world_class_submission_review.md", "world_class_claim_guard": "reports/world_class_claim_guard.md", "registry_audit": "reports/registry_audit.md", "package_verification": "reports/package_verification.md", @@ -837,7 +846,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 91 + "file_count": 92 }, { "path": "evals", @@ -849,10 +858,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 195 + "file_count": 197 } ], - "file_count": 350, + "file_count": 353, "folder_count": 4, "distribution": [ { @@ -877,7 +886,7 @@ }, { "label": "scripts", - "value": 91 + "value": 92 }, { "label": "evals", @@ -885,7 +894,7 @@ }, { "label": "reports", - "value": 195 + "value": 197 } ] }, @@ -1006,7 +1015,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 91 + "file_count": 92 }, { "path": "evals", @@ -1018,7 +1027,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 195 + "file_count": 197 } ], "strengths": [ @@ -1310,9 +1319,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 54 + "changed_file_count": 56 }, - "commit": "b540f580b373cb9c0f575b687fb82c5a16748a5d", + "commit": "30539bed7a29dd88c2f289dd0c115da9834dd5d9", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1387,8 +1396,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 175, - "script_count": 91, + "scanned_files": 176, + "script_count": 92, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -1397,18 +1406,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 61, + "file_write_script_count": 62, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 74, + "help_smoke_checked_count": 75, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 175, - "package_sha256": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac" + "package_hash_file_count": 176, + "package_sha256": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7" }, "skill_atlas": { "skill_count": 12, @@ -1446,8 +1455,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac", - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "package_sha256": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { "openai": "pass", @@ -1478,7 +1487,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1494,8 +1503,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", - "archive_entry_count": 559, + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", + "archive_entry_count": 563, "failure_count": 0, "warning_count": 0 }, @@ -1506,7 +1515,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 559, + "archive_entry_count": 563, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1573,12 +1582,12 @@ { "field": "archive_sha256", "from": "", - "to": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "to": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac" + "to": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7" } ] }, @@ -1595,14 +1604,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1611,7 +1620,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1824,7 +1833,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2056,7 +2065,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ "real external client source", @@ -2084,7 +2093,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -3734,7 +3743,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.92, + "duration_ms": 28.33, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3762,7 +3771,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.93, + "duration_ms": 28.24, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3785,7 +3794,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.97, + "duration_ms": 27.8, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3813,7 +3822,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.94, + "duration_ms": 28.45, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3836,7 +3845,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.93, + "duration_ms": 28.13, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3864,7 +3873,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.69, + "duration_ms": 28.51, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3887,7 +3896,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.79, + "duration_ms": 28.39, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3914,7 +3923,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.09, + "duration_ms": 27.61, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3937,7 +3946,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.97, + "duration_ms": 28.3, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3966,7 +3975,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.84, + "duration_ms": 29.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4412,7 +4421,7 @@ "skill_os2_coverage": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "item_count": 20, @@ -4533,7 +4542,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "91 scripts; secrets 0; help failures 0", + "current": "92 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -4607,7 +4616,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 559; install failures 0", + "current": "archive entries 563; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -5028,7 +5037,7 @@ "compiled_targets": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "target_count": 5, @@ -5231,6 +5240,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -5250,8 +5260,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -5410,7 +5419,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5463,6 +5472,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -5511,14 +5521,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5540,7 +5550,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -5653,7 +5663,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5706,6 +5716,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -5754,14 +5765,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5781,7 +5792,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -6074,6 +6085,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -6093,8 +6105,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -6253,7 +6264,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6306,6 +6317,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -6354,14 +6366,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6383,7 +6395,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -6496,7 +6508,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6549,6 +6561,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -6597,14 +6610,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6624,7 +6637,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -6917,6 +6930,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -6936,8 +6950,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -7096,7 +7109,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7149,6 +7162,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -7197,14 +7211,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7226,7 +7240,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -7332,7 +7346,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7385,6 +7399,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -7433,14 +7448,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7460,7 +7475,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -7744,6 +7759,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -7763,8 +7779,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -7923,7 +7938,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7976,6 +7991,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -8024,14 +8040,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -8053,7 +8069,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -8159,7 +8175,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -8212,6 +8228,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -8260,14 +8277,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -8287,7 +8304,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -8571,6 +8588,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -8590,8 +8608,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -8750,7 +8767,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -8803,6 +8820,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -8851,14 +8869,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -8880,7 +8898,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -8990,7 +9008,7 @@ }, "file_write": { "required": true, - "script_count": 61, + "script_count": 62, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -9043,6 +9061,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -9091,14 +9110,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 74, + "checked_count": 75, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 61, + "file_write_script_count": 62, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -9118,7 +9137,7 @@ ], "capability_counts": { "network": 3, - "file_write": 61, + "file_write": 62, "subprocess": 9, "interactive": 0 }, @@ -9790,8 +9809,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 175, - "script_count": 91, + "scanned_files": 176, + "script_count": 92, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -9800,18 +9819,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 61, + "file_write_script_count": 62, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 74, + "help_smoke_checked_count": 75, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 175, - "package_sha256": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac" + "package_hash_file_count": 176, + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" }, "failures": [], "warnings": [], @@ -10660,6 +10679,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_world_class_submission_review.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a read-only review queue for world-class evidence submissions before ledger acceptance.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/resource_boundary_check.py", "interface": "cli", @@ -11139,9 +11172,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 74, - "checked_count": 74, - "passed_count": 74, + "candidate_count": 75, + "checked_count": 75, + "passed_count": 75, "failed_count": 0, "skipped_count": 17, "failed_scripts": [], @@ -11736,6 +11769,16 @@ "stdout_excerpt": "usage: render_world_class_evidence_plan.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_world_class_submission_review.py", + "command": "python3 scripts/render_world_class_submission_review.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_submission_review.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/resource_boundary_check.py", "command": "python3 scripts/resource_boundary_check.py --help", @@ -12061,6 +12104,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", @@ -12134,11 +12178,11 @@ "python_compatibility": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "root": ".", "summary": { "target_python": "3.11", - "file_count": 150, + "file_count": 152, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -12527,6 +12571,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/render_world_class_submission_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/resource_boundary_check.py", "ok": true, @@ -13049,6 +13099,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_world_class_submission_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_yao_cli.py", "ok": true, @@ -13065,18 +13121,18 @@ "architecture_maintainability": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 147, - "script_file_count": 91, - "test_file_count": 56, + "python_file_count": 149, + "script_file_count": 92, + "test_file_count": 57, "internal_module_count": 19, - "cli_script_count": 73, + "cli_script_count": 74, "command_handler_count": 33, "warn_line_threshold": 900, "block_line_threshold": 1500, - "largest_file_lines": 899, + "largest_file_lines": 898, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" @@ -13084,21 +13140,21 @@ "largest_files": [ { "path": "scripts/render_review_studio.py", - "lines": 899, + "lines": 898, "kind": "cli-script", "severity": "pass", "recommendation": "Move data loading and large section renderers into focused review_studio_* modules." }, { "path": "tests/verify_yao_cli.py", - "lines": 871, + "lines": 891, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { "path": "scripts/yao.py", - "lines": 840, + "lines": 841, "kind": "cli-script", "severity": "pass", "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." @@ -13133,7 +13189,7 @@ }, { "path": "scripts/yao_cli_parser.py", - "lines": 703, + "lines": 714, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." @@ -13182,10 +13238,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1117410, + "other_text_tokens": 1129714, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1118161, - "relevant_file_count": 480, + "estimated_total_text_tokens": 1130465, + "relevant_file_count": 484, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -14108,6 +14164,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -15035,7 +15092,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T22:14:54Z", + "generated_at": "2026-06-13T22:35:30Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -15223,7 +15280,7 @@ "world_class_evidence_ledger": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "ledger_entry_count": 4, @@ -15484,7 +15541,7 @@ "world_class_evidence_intake": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "schema_present": true, @@ -15751,15 +15808,187 @@ "markdown": "reports/world_class_evidence_intake.md" } }, - "world_class_claim_guard": { + "world_class_submission_review": { "schema_version": "1.0", "ok": true, "generated_at": "2026-06-14", "skill_dir": ".", + "summary": { + "review_item_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "valid_packet_source_incomplete_count": 0, + "ready_for_ledger_review_count": 0, + "fix_submission_count": 0, + "unmatched_submission_count": 0, + "invalid_submission_count": 0, + "ready_to_claim_world_class": false, + "review_counts_submission_as_completion": false, + "decision": "awaiting-submissions" + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "review_counts_submission_as_completion": false + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "model_executed_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "accepted": false + }, + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "accepted": false + }, + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ], + "next_action": "Record real A/B choices in the decision template, then regenerate adjudication." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 1, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "unmatched_submissions": [], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json" + }, + "artifacts": { + "json": "reports/world_class_submission_review.json", + "markdown": "reports/world_class_submission_review.md" + } + }, + "world_class_claim_guard": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-13", + "skill_dir": ".", "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 68, + "claim_surface_count": 69, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -16060,6 +16289,10 @@ "path": "reports/world_class_evidence_plan.md", "violation_count": 0 }, + { + "path": "reports/world_class_submission_review.md", + "violation_count": 0 + }, { "path": "reports/yao-deploy-bridge-skill-research-plan.md", "violation_count": 0 @@ -16098,8 +16331,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac", - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { "openai": "pass", @@ -16130,16 +16363,16 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-14" + "generated_at": "2026-06-13" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -16155,7 +16388,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac" + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" } ] }, @@ -16178,8 +16411,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", - "archive_entry_count": 559, + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", + "archive_entry_count": 563, "failure_count": 0, "warning_count": 0 }, @@ -16837,14 +17070,14 @@ "install_simulation": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 559, + "archive_entry_count": 563, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -16862,7 +17095,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -17108,16 +17341,16 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } }, "upgrade_check": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -17186,12 +17419,12 @@ { "field": "archive_sha256", "from": "", - "to": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "to": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac" + "to": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" } ] }, @@ -17206,8 +17439,8 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } }, "manifest": { diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index bce90a7..d49e064 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 175, - "script_count": 91, + "scanned_files": 176, + "script_count": 92, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 61, + "file_write_script_count": 62, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 74, + "help_smoke_checked_count": 75, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 175, - "package_sha256": "4871c36233a53615c8e19c1c59449af60e732fff2d389742ca510422d7cbac75" + "package_hash_file_count": 176, + "package_sha256": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" }, "failures": [], "warnings": [], @@ -872,6 +872,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_world_class_submission_review.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a read-only review queue for world-class evidence submissions before ledger acceptance.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/resource_boundary_check.py", "interface": "cli", @@ -1351,9 +1365,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 74, - "checked_count": 74, - "passed_count": 74, + "candidate_count": 75, + "checked_count": 75, + "passed_count": 75, "failed_count": 0, "skipped_count": 17, "failed_scripts": [], @@ -1948,6 +1962,16 @@ "stdout_excerpt": "usage: render_world_class_evidence_plan.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_world_class_submission_review.py", + "command": "python3 scripts/render_world_class_submission_review.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_submission_review.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/resource_boundary_check.py", "command": "python3 scripts/resource_boundary_check.py --help", @@ -2273,6 +2297,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index ee98399..0807117 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `175` -- Scripts: `91` +- Scanned files: `176` +- Scripts: `92` - Internal script modules: `17` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `61` +- File-write scripts: `62` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `74` +- CLI help smoke checked: `75` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `175` -- Package SHA256: `4871c36233a53615c8e19c1c59449af60e732fff2d389742ca510422d7cbac75` +- Package hash files: `176` +- Package SHA256: `68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `74` -- Passed scripts: `74` +- Checked scripts: `75` +- Passed scripts: `75` - Failed scripts: `none` ## Script Surface @@ -121,6 +121,7 @@ | scripts/render_world_class_evidence_intake.py | cli | True | True | True | False | False | True | False | Validates world-class human and external evidence intake packets before ledger review. | | scripts/render_world_class_evidence_ledger.py | cli | True | True | True | False | False | True | False | Renders a machine-checkable ledger for world-class external and human evidence gaps. | | scripts/render_world_class_evidence_plan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_world_class_submission_review.py | cli | True | True | True | False | False | True | False | Renders a read-only review queue for world-class evidence submissions before ledger acceptance. | | scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/review_studio_data.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to load Review Studio source reports and metric cards. | | scripts/review_studio_formatting.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to format report dictionaries as audit UI panels. | diff --git a/reports/skill-overview.html b/reports/skill-overview.html index edef224..34174cf 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布350项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布353项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index a031873..73812f7 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -436,7 +436,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 91 + "file_count": 92 }, { "path": "evals", @@ -448,10 +448,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 195 + "file_count": 197 } ], - "file_count": 350, + "file_count": 353, "folder_count": 4, "distribution": [ { @@ -476,7 +476,7 @@ }, { "label": "scripts", - "value": 91 + "value": 92 }, { "label": "evals", @@ -484,7 +484,7 @@ }, { "label": "reports", - "value": 195 + "value": 197 } ] }, @@ -605,7 +605,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 91 + "file_count": 92 }, { "path": "evals", @@ -617,7 +617,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 195 + "file_count": 197 } ], "strengths": [ @@ -909,9 +909,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 54 + "changed_file_count": 56 }, - "commit": "b540f580b373cb9c0f575b687fb82c5a16748a5d", + "commit": "30539bed7a29dd88c2f289dd0c115da9834dd5d9", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -986,8 +986,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 175, - "script_count": 91, + "scanned_files": 176, + "script_count": 92, "internal_module_count": 17, "secret_findings": 0, "dependency_files": [ @@ -996,18 +996,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 61, + "file_write_script_count": 62, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 74, + "help_smoke_checked_count": 75, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 175, - "package_sha256": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac" + "package_hash_file_count": 176, + "package_sha256": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7" }, "skill_atlas": { "skill_count": 12, @@ -1045,8 +1045,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac", - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "package_sha256": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, "compatibility": { "openai": "pass", @@ -1077,7 +1077,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1093,8 +1093,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e", - "archive_entry_count": 559, + "archive_sha256": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e", + "archive_entry_count": 563, "failure_count": 0, "warning_count": 0 }, @@ -1105,7 +1105,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 559, + "archive_entry_count": 563, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1172,12 +1172,12 @@ { "field": "archive_sha256", "from": "", - "to": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "to": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "d799ec85c035b6335ea1e262b980d630ec05cd07448239cf97a34540eb4c2bac" + "to": "6298aa08d70a1e36956df32bcbb7a0bc778e8125f5449630cebf966b1c06a5f7" } ] }, @@ -1194,14 +1194,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1210,7 +1210,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1423,7 +1423,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1655,7 +1655,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ "real external client source", @@ -1683,7 +1683,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index eb73abc..3bcbf5c 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -123,6 +123,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 70038c1..fa98027 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 91; help failures 0", + "current": "secrets 0; scripts 92; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -281,7 +281,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 559; install failures 0; permission failures 0", + "current": "zip entries 563; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index fb8d0e2..6fe6d1a 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,11 +23,11 @@ Generated at: `2026-06-13` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 20; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 91; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 92; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4 | Packaged adapters expose explicit permission metadata and residual risks | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0 | At least one target/client enforces approved permissions at runtime | Integrate a real client or installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 559; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 563; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 7dc94d4..79a915d 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -122,7 +122,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "91 scripts; secrets 0; help failures 0", + "current": "92 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -196,7 +196,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 559; install failures 0", + "current": "archive entries 563; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index ba971ed..ae1c6a4 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -21,9 +21,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 91 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 92 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 559; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 563; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index b29c797..1be1718 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "fe47e53a7a89918ef2d7ae53c1e62a6b73da0bd80904d5a157c71a2758c8004e" + "to": "cb1ac8c62bc5dee942a88f18eb5d71c93ae58f854ac19f3d912f9c9c3a4ec24e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "4871c36233a53615c8e19c1c59449af60e732fff2d389742ca510422d7cbac75" + "to": "68a0228b56de8c9e9e94d64d7c1e386bf3dff55c0f6228b65f91e75d02525926" } ] }, diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index 4080935..3aa27bd 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -6,7 +6,7 @@ "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 68, + "claim_surface_count": 69, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -307,6 +307,10 @@ "path": "reports/world_class_evidence_plan.md", "violation_count": 0 }, + { + "path": "reports/world_class_submission_review.md", + "violation_count": 0 + }, { "path": "reports/yao-deploy-bridge-skill-research-plan.md", "violation_count": 0 diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index aca98a6..1c4e190 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -7,7 +7,7 @@ Generated at: `2026-06-13` - decision: `claim-guard-pass-evidence-pending` - ledger ready to claim world-class: `false` - ledger pending evidence: `4` -- claim surfaces scanned: `68` +- claim surfaces scanned: `69` - violations: `0` - overclaim guard active: `true` diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json new file mode 100644 index 0000000..59be3fb --- /dev/null +++ b/reports/world_class_submission_review.json @@ -0,0 +1,172 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-13", + "skill_dir": ".", + "summary": { + "review_item_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "valid_packet_source_incomplete_count": 0, + "ready_for_ledger_review_count": 0, + "fix_submission_count": 0, + "unmatched_submission_count": 0, + "invalid_submission_count": 0, + "ready_to_claim_world_class": false, + "review_counts_submission_as_completion": false, + "decision": "awaiting-submissions" + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "review_counts_submission_as_completion": false + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "model_executed_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "accepted": false + }, + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "accepted": false + }, + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ], + "next_action": "Record real A/B choices in the decision template, then regenerate adjudication." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 1, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "unmatched_submissions": [], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json" + }, + "artifacts": { + "json": "reports/world_class_submission_review.json", + "markdown": "reports/world_class_submission_review.md" + } +} diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md new file mode 100644 index 0000000..3b8b743 --- /dev/null +++ b/reports/world_class_submission_review.md @@ -0,0 +1,137 @@ +# World-Class Submission Review + +Generated at: `2026-06-13` + +## Summary + +- decision: `awaiting-submissions` +- review items: `4` +- accepted: `0` +- awaiting submission: `4` +- valid packet but source incomplete: `0` +- ready for ledger review: `0` +- fix submission: `0` +- unmatched submissions: `0` +- ready to claim world-class: `false` +- review counts submission as completion: `false` + +This report is a read-only reviewer queue. It does not accept evidence or make world-class completion true. + +## Queue + +| Evidence | Review state | Intake | Source accepted | Submission | Next action | +| --- | --- | --- | --- | --- | --- | +| `provider-holdout` | `awaiting-submission` | `missing` | `false` | `missing` | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| `human-adjudication` | `awaiting-submission` | `missing` | `false` | `missing` | Record real A/B choices in the decision template, then regenerate adjudication. | +| `native-permission-enforcement` | `awaiting-submission` | `missing` | `false` | `missing` | Integrate a real client or installer runtime guard before claiming native permission enforcement. | +| `native-client-telemetry` | `awaiting-submission` | `missing` | `false` | `missing` | Install a real client against the native host and import production metadata-only events. | + +## Details + +### Provider Holdout + +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. +- ledger status: `pending` +- submission status: `missing` +- intake status: `missing` +- source accepted: `false` +- submission path: `evidence/world_class/submissions/provider-holdout.json` + +#### Source Checks + +- reports/output_execution_runs.json summary.model_executed_count > 0 +- reports/output_execution_runs.json summary.timing_observed_count > 0 +- reports/output_execution_runs.json summary.token_observed_count > 0 +- reports/skill_os2_audit.json item provider-holdout status becomes pass + +#### Intake Errors + +- No intake errors. + +#### Privacy Contract + +- Do not commit provider credentials or environment dumps. +- The output execution report records output hashes and aggregate run metadata, not raw provider prompts. + +### Human Adjudication + +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. +- ledger status: `pending` +- submission status: `missing` +- intake status: `missing` +- source accepted: `false` +- submission path: `evidence/world_class/submissions/human-adjudication.json` + +#### Source Checks + +- reports/output_review_adjudication.json summary.pending_count == 0 +- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count +- reports/output_review_adjudication.json summary.invalid_decision_count == 0 +- reports/skill_os2_audit.json item human-adjudication status becomes pass + +#### Intake Errors + +- No intake errors. + +#### Privacy Contract + +- Reviewer decisions should not include raw user data or private customer detail. +- Keep the answer key separate until after decisions are recorded. + +### Native Permission Enforcement + +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. +- ledger status: `pending` +- submission status: `missing` +- intake status: `missing` +- source accepted: `false` +- submission path: `evidence/world_class/submissions/native-permission-enforcement.json` + +#### Source Checks + +- reports/runtime_permission_probes.json summary.native_enforcement_count > 0 +- reports/runtime_permission_probes.json summary.failure_count == 0 +- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass + +#### Intake Errors + +- No intake errors. + +#### Privacy Contract + +- Do not mark native_enforcement true for metadata-only fallbacks. +- Keep residual risks visible for targets that still rely on operator enforcement. + +### Native Client Telemetry + +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. +- ledger status: `pending` +- submission status: `missing` +- intake status: `missing` +- source accepted: `false` +- submission path: `evidence/world_class/submissions/native-client-telemetry.json` + +#### Source Checks + +- reports/adoption_drift_report.json summary.source_types.external > 0 +- reports/adoption_drift_report.json summary.adoption_sample_count > 0 +- reports/skill_os2_audit.json item native-client-telemetry status becomes pass + +#### Intake Errors + +- No intake errors. + +#### Privacy Contract + +- Telemetry must remain metadata-only and local-first. +- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. + +## Boundary + +- A valid submission packet is not accepted evidence by itself. +- Planned work, metadata fallback, pending human review, and local command-runner output still do not count. +- The world-class ledger remains the source of truth for `ready_to_claim_world_class`. diff --git a/scripts/ci_test.py b/scripts/ci_test.py index fdb8521..6167e87 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -56,6 +56,7 @@ DEFAULT_TARGETS = [ "world-class-evidence-check", "world-class-ledger-check", "world-class-intake-check", + "world-class-submission-review-check", "world-class-claim-guard-check", "benchmark-reproducibility-check", "feedback-check", diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index c8bbd5b..617ee41 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -264,11 +264,12 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = { "world-class-evidence": { "summary": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。", "why": "世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。", - "source_fix": "reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md", + "source_fix": "reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md", "source_paths": [ {"path": "reports/world_class_evidence_ledger.md", "label": "world-class evidence ledger", "kind": "report", "patterns": ["# World-Class Evidence Ledger"]}, {"path": "reports/world_class_evidence_plan.md", "label": "world-class evidence plan", "kind": "report", "patterns": ["# World-Class Evidence Plan"]}, {"path": "reports/world_class_evidence_intake.md", "label": "world-class evidence intake", "kind": "report", "patterns": ["# World-Class Evidence Intake"]}, + {"path": "reports/world_class_submission_review.md", "label": "world-class submission review", "kind": "report", "patterns": ["# World-Class Submission Review"]}, {"path": "reports/world_class_claim_guard.md", "label": "world-class claim guard", "kind": "report", "patterns": ["# World-Class Claim Guard"]}, {"path": "evidence/world_class/intake.schema.json", "label": "evidence intake schema", "kind": "schema", "patterns": ["Yao World-Class Evidence Intake"]}, {"path": "evidence/world_class/templates/provider-holdout.intake.json", "label": "provider intake template", "kind": "template", "patterns": ["provider-holdout"]}, @@ -278,7 +279,7 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = { {"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]}, {"path": "reports/adoption_drift_report.md", "label": "adoption drift", "kind": "report", "patterns": ["# Adoption"]}, ], - "verification": "python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .", + "verification": "python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .", }, "registry-audit": { "summary": "补齐 registry package metadata、checksum、license、owner、review cadence 和 install evidence。", @@ -887,11 +888,9 @@ def main() -> None: parser.add_argument("--output-html") parser.add_argument("--output-json") args = parser.parse_args() - payload = render_review_studio( - Path(args.skill_dir), - output_html=Path(args.output_html).resolve() if args.output_html else None, - output_json=Path(args.output_json).resolve() if args.output_json else None, - ) + output_html = Path(args.output_html).resolve() if args.output_html else None + output_json = Path(args.output_json).resolve() if args.output_json else None + payload = render_review_studio(Path(args.skill_dir), output_html=output_html, output_json=output_json) print(json.dumps(payload, ensure_ascii=False, indent=2)) diff --git a/scripts/render_world_class_submission_review.py b/scripts/render_world_class_submission_review.py new file mode 100644 index 0000000..cd0c235 --- /dev/null +++ b/scripts/render_world_class_submission_review.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +from render_world_class_evidence_intake import build_intake +from render_world_class_evidence_ledger import build_ledger + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a read-only review queue for world-class evidence submissions before ledger acceptance." + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def by_key(items: list[dict[str, Any]], key_name: str = "evidence_key") -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = {} + for item in items: + key = str(item.get(key_name, "")) + if key and key not in result: + result[key] = item + return result + + +def review_state(entry: dict[str, Any], intake_result: dict[str, Any] | None) -> tuple[str, str]: + if entry.get("status") == "accepted": + return "accepted", "Ledger already accepts this evidence item." + + submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {} + submission_status = str(submission.get("status", "missing")) + if submission_status == "missing": + return "awaiting-submission", "No evidence packet has been submitted for review." + if submission_status in {"invalid-json", "invalid-contract"}: + return "fix-submission", "Submission exists but fails the ledger submission contract." + if intake_result is None: + return "fix-submission", "Submission exists but is missing from the intake validation results." + if intake_result.get("status") != "pass": + return "fix-submission", "Submission exists but failed intake validation." + + observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {} + if observed.get("accepted") is True: + return "ready-for-ledger-review", "Submission and source evidence are ready for final ledger review." + return "source-evidence-incomplete", "Submission packet is valid, but the source evidence checks still do not pass." + + +def build_item(entry: dict[str, Any], intake_result: dict[str, Any] | None) -> dict[str, Any]: + state, reason = review_state(entry, intake_result) + submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {} + observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {} + return { + "evidence_key": entry.get("key", ""), + "label": entry.get("label", entry.get("key", "")), + "category": entry.get("category", "external"), + "owner": entry.get("owner", "release reviewer"), + "ledger_status": entry.get("status", "pending"), + "submission_status": submission.get("status", "missing"), + "intake_status": intake_result.get("status", "missing") if intake_result else "missing", + "source_accepted": observed.get("accepted") is True, + "review_state": state, + "blocking_reason": reason, + "submission_path": submission.get("path", ""), + "submitted_by": submission.get("submitted_by", ""), + "submitted_at": submission.get("submitted_at", ""), + "artifact_ref_count": submission.get("artifact_ref_count", 0), + "intake_errors": intake_result.get("errors", []) if intake_result else [], + "observed_state": observed, + "success_checks": entry.get("success_checks", []), + "privacy_contract": entry.get("privacy_contract", []), + "next_action": entry.get("next_action", ""), + } + + +def build_submission_review(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir) + intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir) + intake_by_key = by_key(intake.get("submissions", [])) + entries = ledger.get("entries", []) + items = [build_item(entry, intake_by_key.get(str(entry.get("key", "")))) for entry in entries] + known_keys = {str(entry.get("key", "")) for entry in entries} + unmatched = [item for item in intake.get("submissions", []) if str(item.get("evidence_key", "")) not in known_keys] + counts: dict[str, int] = {} + for item in items: + state = str(item.get("review_state", "unknown")) + counts[state] = counts.get(state, 0) + 1 + invalid_count = counts.get("fix-submission", 0) + len(unmatched) + ready_count = counts.get("ready-for-ledger-review", 0) + accepted_count = counts.get("accepted", 0) + source_incomplete_count = counts.get("source-evidence-incomplete", 0) + awaiting_count = counts.get("awaiting-submission", 0) + if accepted_count == len(items) and items: + decision = "ledger-complete" + elif invalid_count: + decision = "fix-submissions" + elif ready_count: + decision = "review-ready" + elif source_incomplete_count: + decision = "source-evidence-incomplete" + else: + decision = "awaiting-submissions" + return { + "schema_version": "1.0", + "ok": invalid_count == 0, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": { + "review_item_count": len(items), + "accepted_count": accepted_count, + "awaiting_submission_count": awaiting_count, + "valid_packet_source_incomplete_count": source_incomplete_count, + "ready_for_ledger_review_count": ready_count, + "fix_submission_count": counts.get("fix-submission", 0), + "unmatched_submission_count": len(unmatched), + "invalid_submission_count": invalid_count, + "ready_to_claim_world_class": ledger.get("summary", {}).get("ready_to_claim_world_class") is True, + "review_counts_submission_as_completion": False, + "decision": decision, + }, + "submissions": { + "directory": rel_path(submissions_dir, skill_dir), + "review_counts_submission_as_completion": False, + }, + "items": items, + "unmatched_submissions": unmatched, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + }, + "artifacts": { + "json": "reports/world_class_submission_review.json", + "markdown": "reports/world_class_submission_review.md", + }, + } + + +def render_list(values: list[Any], empty: str) -> list[str]: + if not values: + return [f"- {empty}"] + return [f"- {value}" for value in values] + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# World-Class Submission Review", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- review items: `{summary['review_item_count']}`", + f"- accepted: `{summary['accepted_count']}`", + f"- awaiting submission: `{summary['awaiting_submission_count']}`", + f"- valid packet but source incomplete: `{summary['valid_packet_source_incomplete_count']}`", + f"- ready for ledger review: `{summary['ready_for_ledger_review_count']}`", + f"- fix submission: `{summary['fix_submission_count']}`", + f"- unmatched submissions: `{summary['unmatched_submission_count']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- review counts submission as completion: `{str(summary['review_counts_submission_as_completion']).lower()}`", + "", + "This report is a read-only reviewer queue. It does not accept evidence or make world-class completion true.", + "", + "## Queue", + "", + "| Evidence | Review state | Intake | Source accepted | Submission | Next action |", + "| --- | --- | --- | --- | --- | --- |", + ] + for item in report["items"]: + source_accepted = str(item.get("source_accepted") is True).lower() + next_action = str(item.get("next_action", "")).replace("|", "\\|") + lines.append( + f"| `{item['evidence_key']}` | `{item['review_state']}` | `{item['intake_status']}` | " + f"`{source_accepted}` | `{item['submission_status']}` | {next_action} |" + ) + if report.get("unmatched_submissions"): + lines.extend(["", "## Unmatched Submissions", ""]) + for item in report["unmatched_submissions"]: + errors = "; ".join(item.get("errors", [])) or "unknown evidence key" + lines.append(f"- `{item.get('path', '')}`: {errors}") + lines.extend(["", "## Details", ""]) + for item in report["items"]: + lines.extend( + [ + f"### {item['label']}", + "", + f"- review state: `{item['review_state']}`", + f"- blocking reason: {item['blocking_reason']}", + f"- ledger status: `{item['ledger_status']}`", + f"- submission status: `{item['submission_status']}`", + f"- intake status: `{item['intake_status']}`", + f"- source accepted: `{str(item['source_accepted']).lower()}`", + f"- submission path: `{item.get('submission_path') or 'missing'}`", + "", + "#### Source Checks", + "", + *render_list(item.get("success_checks", []), "No source checks listed."), + "", + "#### Intake Errors", + "", + *render_list(item.get("intake_errors", []), "No intake errors."), + "", + "#### Privacy Contract", + "", + *render_list(item.get("privacy_contract", []), "No privacy contract listed."), + "", + ] + ) + lines.extend( + [ + "## Boundary", + "", + "- A valid submission packet is not accepted evidence by itself.", + "- Planned work, metadata fallback, pending human review, and local command-runner output still do not count.", + "- The world-class ledger remains the source of truth for `ready_to_claim_world_class`.", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a read-only review queue for world-class evidence submissions.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--submissions-dir") + parser.add_argument("--output-json", default="reports/world_class_submission_review.json") + parser.add_argument("--output-md", default="reports/world_class_submission_review.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None + report = build_submission_review(skill_dir, args.generated_at, submissions_dir=submissions_dir) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/review_studio_data.py b/scripts/review_studio_data.py index fb65fcb..d12e784 100644 --- a/scripts/review_studio_data.py +++ b/scripts/review_studio_data.py @@ -81,6 +81,7 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]: "world_class_evidence_plan": "reports/world_class_evidence_plan.md", "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", "world_class_evidence_intake": "reports/world_class_evidence_intake.md", + "world_class_submission_review": "reports/world_class_submission_review.md", "world_class_claim_guard": "reports/world_class_claim_guard.md", "registry_audit": "reports/registry_audit.md", "package_verification": "reports/package_verification.md", @@ -118,6 +119,7 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: "review_annotations": load_json(reports / "review_annotations.json"), "world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"), "world_class_evidence_intake": load_json(reports / "world_class_evidence_intake.json"), + "world_class_submission_review": load_json(reports / "world_class_submission_review.json"), "world_class_claim_guard": load_json(reports / "world_class_claim_guard.json"), "registry": load_json(reports / "registry_audit.json"), "package_verification": load_json(reports / "package_verification.json"), diff --git a/scripts/yao.py b/scripts/yao.py index 7b0b24a..c8202bd 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -35,6 +35,7 @@ from yao_cli_report_commands import ( command_world_class_intake, command_world_class_ledger, command_world_class_submission_kit, + command_world_class_submission_review, ) from yao_cli_runtime import ROOT, run_script from yao_cli_telemetry import add_telemetry_args, maybe_record_cli_event diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index a0300c7..8b3977b 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -219,6 +219,17 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] world_class_submission_kit_cmd.add_argument("--generated-at") world_class_submission_kit_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_submission_kit")) + world_class_submission_review_cmd = subparsers.add_parser( + "world-class-submission-review", + help="Render a read-only review queue for world-class evidence submissions.", + ) + world_class_submission_review_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_submission_review_cmd.add_argument("--submissions-dir") + world_class_submission_review_cmd.add_argument("--output-json") + world_class_submission_review_cmd.add_argument("--output-md") + world_class_submission_review_cmd.add_argument("--generated-at") + world_class_submission_review_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_submission_review")) + world_class_claim_guard_cmd = subparsers.add_parser( "world-class-claim-guard", help="Scan public claim surfaces for premature world-class completion claims.", diff --git a/scripts/yao_cli_report_commands.py b/scripts/yao_cli_report_commands.py index a6cd3d0..fa59c5b 100644 --- a/scripts/yao_cli_report_commands.py +++ b/scripts/yao_cli_report_commands.py @@ -90,6 +90,14 @@ def command_world_class_submission_kit(args: argparse.Namespace) -> int: return emit_result(run_script("prepare_world_class_submission_kit.py", cmd)) +def command_world_class_submission_review(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.submissions_dir: + cmd.extend(["--submissions-dir", args.submissions_dir]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_submission_review.py", cmd)) + + def command_world_class_claim_guard(args: argparse.Namespace) -> int: cmd = [resolved_skill_dir(args)] for surface in args.claim_surface: diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index e21ee3c..67f3066 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -167,6 +167,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", @@ -186,8 +187,7 @@ "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", - "scripts/telemetry_native_host.py", - "scripts/trigger_eval.py" + "scripts/telemetry_native_host.py" ], "assets": [ "templates/basic_skill.md.j2", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index ae65efd..c481c6b 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -79,6 +79,7 @@ "scripts/render_world_class_evidence_intake.py", "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 25d7802..f7fde2d 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -243,6 +243,7 @@ def main() -> None: "render_world_class_evidence_plan.py", "render_world_class_evidence_ledger.py", "render_world_class_evidence_intake.py", + "render_world_class_submission_review.py", "render_world_class_claim_guard.py", "render_skill_os2_coverage.py", ]: @@ -374,6 +375,8 @@ def main() -> None: assert full_payload["evidence_paths"]["world_class_evidence_ledger"] == "reports/world_class_evidence_ledger.md", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_evidence_intake.md").exists(): assert full_payload["evidence_paths"]["world_class_evidence_intake"] == "reports/world_class_evidence_intake.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_submission_review.md").exists(): + assert full_payload["evidence_paths"]["world_class_submission_review"] == "reports/world_class_submission_review.md", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_claim_guard.md").exists(): assert full_payload["evidence_paths"]["world_class_claim_guard"] == "reports/world_class_claim_guard.md", full_payload["evidence_paths"] assert full_payload["data"]["output_blind_review"]["summary"]["pair_count"] == 5, full_payload["data"]["output_blind_review"] @@ -407,6 +410,7 @@ def main() -> None: "reports/world_class_evidence_ledger.md", "reports/world_class_evidence_plan.md", "reports/world_class_evidence_intake.md", + "reports/world_class_submission_review.md", "reports/world_class_claim_guard.md", "evidence/world_class/intake.schema.json", "evidence/world_class/templates/provider-holdout.intake.json", @@ -414,8 +418,12 @@ def main() -> None: "reports/skill_os2_audit.md", }, world_class_action assert all(item["exists"] for item in world_class_action["source_refs"]), world_class_action + assert "world-class-submission-review" in world_class_action["verification_command"], world_class_action assert full_payload["data"]["world_class_evidence_ledger"]["summary"]["pending_count"] == 4, full_payload["data"]["world_class_evidence_ledger"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_evidence_intake"] + assert full_payload["data"]["world_class_submission_review"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_submission_review"] + assert full_payload["data"]["world_class_submission_review"]["summary"]["review_counts_submission_as_completion"] is False, full_payload["data"]["world_class_submission_review"] + assert full_payload["data"]["world_class_submission_review"]["summary"]["awaiting_submission_count"] == 4, full_payload["data"]["world_class_submission_review"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["template_pass_count"] == 4, full_payload["data"]["world_class_evidence_intake"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["operator_checklist_count"] == 4, full_payload["data"]["world_class_evidence_intake"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["operator_checklist_ready_count"] == 0, full_payload["data"]["world_class_evidence_intake"] diff --git a/tests/verify_world_class_submission_review.py b/tests/verify_world_class_submission_review.py new file mode 100644 index 0000000..b0198f7 --- /dev/null +++ b/tests/verify_world_class_submission_review.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_submission_review.py" +TMP = ROOT / "tests" / "tmp_world_class_submission_review" + + +def provider_submission(*, valid: bool = True) -> dict: + return { + "schema_version": "1.0", + "evidence_key": "provider-holdout", + "template_only": False, + "category": "external", + "source_type": "provider-output-eval", + "submitted_by": "Yao provider operator", + "submitted_at": "2026-06-14", + "summary": "Aggregate provider-backed holdout evidence for ledger review.", + "artifact_refs": [ + { + "path": "reports/output_execution_runs.json", + "kind": "aggregate-report", + "contains_raw_content": not valid, + } + ], + "provenance": { + "provider": "openai", + "model": "gpt-4.1-mini", + "credential_material_committed": False, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": valid, + "reviewer_or_operator_identity_present": valid, + "artifact_refs_reviewed": valid, + "privacy_contract_satisfied": valid, + }, + } + + +def run_review(*extra: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra, + ], + cwd=ROOT, + capture_output=True, + text=True, + check=check, + ) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + + output_json = TMP / "world_class_submission_review.json" + output_md = TMP / "world_class_submission_review.md" + default_proc = run_review("--output-json", str(output_json), "--output-md", str(output_md)) + payload = json.loads(default_proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["decision"] == "awaiting-submissions", summary + assert summary["review_item_count"] == 4, summary + assert summary["awaiting_submission_count"] == 4, summary + assert summary["ready_for_ledger_review_count"] == 0, summary + assert summary["valid_packet_source_incomplete_count"] == 0, summary + assert summary["review_counts_submission_as_completion"] is False, summary + provider_item = {item["evidence_key"]: item for item in payload["items"]}["provider-holdout"] + assert provider_item["review_state"] == "awaiting-submission", provider_item + assert provider_item["submission_status"] == "missing", provider_item + assert provider_item["source_accepted"] is False, provider_item + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Submission Review" in markdown, markdown + assert "review counts submission as completion: `false`" in markdown, markdown + assert "`provider-holdout`" in markdown, markdown + + submissions = TMP / "valid_submissions" + submissions.mkdir() + (submissions / "provider-holdout.json").write_text( + json.dumps(provider_submission(valid=True), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + # The kit manifest must not be treated as an evidence packet. + (submissions / "submission_manifest.json").write_text( + json.dumps({"schema_version": "1.0", "not": "an evidence packet"}, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + submitted_proc = run_review( + "--submissions-dir", + str(submissions), + "--output-json", + str(TMP / "submitted_review.json"), + "--output-md", + str(TMP / "submitted_review.md"), + ) + submitted_payload = json.loads(submitted_proc.stdout) + submitted_summary = submitted_payload["summary"] + assert submitted_summary["decision"] == "source-evidence-incomplete", submitted_summary + assert submitted_summary["valid_packet_source_incomplete_count"] == 1, submitted_summary + assert submitted_summary["awaiting_submission_count"] == 3, submitted_summary + assert submitted_summary["invalid_submission_count"] == 0, submitted_summary + submitted_provider = {item["evidence_key"]: item for item in submitted_payload["items"]}["provider-holdout"] + assert submitted_provider["review_state"] == "source-evidence-incomplete", submitted_provider + assert submitted_provider["intake_status"] == "pass", submitted_provider + assert submitted_provider["submission_status"] == "submitted", submitted_provider + assert submitted_provider["source_accepted"] is False, submitted_provider + assert "model_executed_count" in submitted_provider["observed_state"], submitted_provider + + invalid_dir = TMP / "invalid_submissions" + invalid_dir.mkdir() + (invalid_dir / "provider-holdout.json").write_text( + json.dumps(provider_submission(valid=False), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + invalid_proc = run_review( + "--submissions-dir", + str(invalid_dir), + "--output-json", + str(TMP / "invalid_review.json"), + "--output-md", + str(TMP / "invalid_review.md"), + check=False, + ) + assert invalid_proc.returncode == 2, invalid_proc.stdout + invalid_payload = json.loads(invalid_proc.stdout) + assert invalid_payload["ok"] is False, invalid_payload + assert invalid_payload["summary"]["decision"] == "fix-submissions", invalid_payload["summary"] + invalid_provider = {item["evidence_key"]: item for item in invalid_payload["items"]}["provider-holdout"] + assert invalid_provider["review_state"] == "fix-submission", invalid_provider + assert invalid_provider["intake_errors"], invalid_provider + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 45536c6..fa40b8b 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -86,6 +86,7 @@ def main() -> None: assert "world-class-ledger" in parser_help, parser_help assert "world-class-intake" in parser_help, parser_help assert "world-class-submission-kit" in parser_help, parser_help + assert "world-class-submission-review" in parser_help, parser_help assert "world-class-claim-guard" in parser_help, parser_help assert "benchmark-reproducibility" in parser_help, parser_help assert "telemetry-import" in parser_help, parser_help @@ -294,6 +295,25 @@ def main() -> None: assert (tmp_root / "world_class_submission_kit" / "provider-holdout.json").exists(), world_class_submission_kit_result assert (tmp_root / "world_class_submission_kit" / "submission_manifest.json").exists(), world_class_submission_kit_result + world_class_submission_review_result = run( + "world-class-submission-review", + str(ROOT), + "--submissions-dir", + str(tmp_root / "world_class_submission_kit"), + "--output-json", + str(tmp_root / "world_class_submission_review.json"), + "--output-md", + str(tmp_root / "world_class_submission_review.md"), + "--generated-at", + "2026-06-14", + ) + assert not world_class_submission_review_result["ok"], world_class_submission_review_result + assert world_class_submission_review_result["returncode"] == 2, world_class_submission_review_result + assert world_class_submission_review_result["payload"]["summary"]["decision"] == "fix-submissions", world_class_submission_review_result + assert world_class_submission_review_result["payload"]["summary"]["invalid_submission_count"] == 1, world_class_submission_review_result + assert world_class_submission_review_result["payload"]["items"][0]["review_state"] == "fix-submission", world_class_submission_review_result + assert (tmp_root / "world_class_submission_review.md").exists(), world_class_submission_review_result + world_class_claim_guard_result = run( "world-class-claim-guard", str(ROOT),
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder