diff --git a/Makefile b/Makefile index c920c599..f89e090c 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adaptation-safety-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -127,6 +127,9 @@ world-class-claim-guard-check: benchmark-reproducibility-check: $(PYTHON) tests/verify_benchmark_reproducibility.py +evidence-consistency-check: + $(PYTHON) tests/verify_evidence_consistency.py + feedback-check: $(PYTHON) tests/verify_feedback.py @@ -244,11 +247,11 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adaptation-safety-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_interpretation tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_operator_runbook tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_interpretation tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_operator_runbook tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_evidence_consistency tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 14f2ecaf..18e56a5d 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a world-class operator runbook that gives reviewers the exact commands, artifacts, and collection checklist needed to close remaining evidence gaps - a world-class claim guard that scans public claim surfaces and blocks premature completed/true claims while the evidence ledger still has pending external or human evidence - a benchmark reproducibility manifest that checks methodology sections, required artifacts, failure disclosure, and reproduction commands +- an evidence consistency gate that compares generated reports against each other so benchmark, overview, interpretation, adoption, world-class ledger, coverage, and Review Studio facts do not drift silently - Output Eval Lab evidence with assertion grading, execution/timing/token evidence, a blind A/B review pack, a separate answer key, and reviewer adjudication reports - a runtime permission probe report that checks packaged target adapters for explicit permission metadata, native-enforcement flags, metadata fallback notes, and residual risks - a Python compatibility gate that catches supported-runtime syntax hazards before they reach GitHub Actions or packaged distribution @@ -157,6 +158,7 @@ python3 scripts/yao.py world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR" python3 scripts/yao.py world-class-runbook . --submissions-dir "$SUBMISSIONS_DIR" python3 scripts/yao.py world-class-claim-guard . python3 scripts/yao.py benchmark-reproducibility . +python3 scripts/yao.py evidence-consistency . ``` ## Local Development Source @@ -309,6 +311,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - world-class operator runbook is published in [reports/world_class_operator_runbook.md](reports/world_class_operator_runbook.md) and [reports/world_class_operator_runbook.html](reports/world_class_operator_runbook.html) - world-class public claim guard status is published in [reports/world_class_claim_guard.md](reports/world_class_claim_guard.md) - benchmark reproducibility evidence is published in [reports/benchmark_reproducibility.md](reports/benchmark_reproducibility.md) +- cross-report evidence consistency is published in [reports/evidence_consistency.md](reports/evidence_consistency.md) - target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md) - Python runtime compatibility evidence is published in [reports/python_compatibility.md](reports/python_compatibility.md) - registry package metadata and audit status are published in [reports/registry_audit.md](reports/registry_audit.md) @@ -425,7 +428,7 @@ Utility scripts that make the meta-skill operational: - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs - `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence -- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, python-compat, review, release-snapshot, workspace-flow, report, skill-report, skill-interpretation, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, world-class-submission-kit, world-class-submission-review, world-class-runbook, world-class-claim-guard, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint +- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, python-compat, review, release-snapshot, workspace-flow, report, skill-report, skill-interpretation, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, world-class-submission-kit, world-class-submission-review, world-class-runbook, world-class-claim-guard, benchmark-reproducibility, evidence-consistency, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot - `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger @@ -443,6 +446,7 @@ Utility scripts that make the meta-skill operational: - `render_world_class_operator_runbook.py`: renders an operator-facing checklist and command map for collecting pending world-class evidence without accepting evidence - `render_world_class_claim_guard.py`: scans README, docs, and reports for premature world-class completion claims while accepted evidence is still pending - `render_benchmark_reproducibility.py`: renders methodology, artifact, failure-disclosure, and reproduction-command evidence for public benchmark claims +- `render_evidence_consistency.py`: compares generated report facts across benchmark reproducibility, overview, interpretation, adoption drift, world-class ledger, coverage, and Review Studio artifacts - `python_compat_check.py`: checks Python source for supported-runtime compatibility hazards such as Python 3.11 f-string expression backslashes - `cross_packager.py`: builds client-specific export artifacts from Skill IR plus neutral metadata, with explicit platform contracts and validation - `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage diff --git a/registry/index.json b/registry/index.json index cc05cc19..2cca4c5b 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58" + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index aa76fe8a..4be5e83a 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,7 +16,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58", + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc" }, "compatibility": { diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index bbd0e51d..72d66795 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-15T13:34:11Z", + "generated_at": "2026-06-15T13:58:27Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -25,14 +25,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -41,7 +41,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -53,31 +53,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "skill_activation", + "event": "review_event", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } diff --git a/reports/adoption_drift_report.md b/reports/adoption_drift_report.md index 206372c9..91b31983 100644 --- a/reports/adoption_drift_report.md +++ b/reports/adoption_drift_report.md @@ -5,9 +5,9 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Summary - Events: `1` -- Adoption samples: `1` -- Activation events: `1` -- Adoption rate: `100.0` +- Adoption samples: `0` +- Activation events: `0` +- Adoption rate: `0` - Missed trigger signals: `0` - Bad output signals: `0` - Script error signals: `0` @@ -25,7 +25,7 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, | Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate | | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| `yao-meta-skill` | 1 | 1 | 1 | 0 | 0 | 0 | 100.0 | +| `yao-meta-skill` | 1 | 0 | 0 | 0 | 0 | 0 | 0 | ## Next Iteration Candidates @@ -33,4 +33,4 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Recent Metadata Events -- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` source=`manual` command=`unknown` activation=`explicit` outcome=`accepted` failure=`none` +- `2026-06-13T12:00:00Z` `yao-meta-skill` event=`review_event` source=`manual` command=`unknown` activation=`manual` outcome=`reviewed` failure=`none` diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index 33df3875..12fd0d1d 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -1,37 +1,37 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { - "python_file_count": 168, - "script_file_count": 106, - "test_file_count": 62, + "python_file_count": 170, + "script_file_count": 107, + "test_file_count": 63, "internal_module_count": 29, - "cli_script_count": 79, + "cli_script_count": 80, "command_handler_count": 34, "warn_line_threshold": 900, "block_line_threshold": 1500, - "largest_file_lines": 886, + "largest_file_lines": 896, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" }, "largest_files": [ - { - "path": "scripts/yao.py", - "lines": 886, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." - }, { "path": "tests/verify_yao_cli.py", - "lines": 886, + "lines": 896, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, + { + "path": "scripts/yao.py", + "lines": 889, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." + }, { "path": "scripts/skill_report_layout.py", "lines": 808, @@ -48,7 +48,7 @@ }, { "path": "scripts/yao_cli_parser.py", - "lines": 774, + "lines": 784, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 7d847b7c..86e9fe46 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -1,17 +1,17 @@ # Architecture Maintainability -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary - decision: `pass` -- python files: `168` -- scripts: `106` -- tests: `62` +- python files: `170` +- scripts: `107` +- tests: `63` - internal modules: `29` -- CLI scripts: `79` +- CLI scripts: `80` - Yao CLI command handlers: `34` -- largest file lines: `886` +- largest file lines: `896` - hotspots: `0` - blockers: `0` @@ -25,11 +25,11 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | -| `scripts/yao.py` | `886` | `cli-script` | `pass` | -| `tests/verify_yao_cli.py` | `886` | `test` | `pass` | +| `tests/verify_yao_cli.py` | `896` | `test` | `pass` | +| `scripts/yao.py` | `889` | `cli-script` | `pass` | | `scripts/skill_report_layout.py` | `808` | `internal-module` | `pass` | | `scripts/skill_report_model.py` | `801` | `internal-module` | `pass` | -| `scripts/yao_cli_parser.py` | `774` | `internal-module` | `pass` | +| `scripts/yao_cli_parser.py` | `784` | `internal-module` | `pass` | | `tests/verify_review_studio.py` | `751` | `test` | `pass` | | `scripts/compile_skill.py` | `734` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `730` | `cli-script` | `pass` | diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 84bd8eab..58502824 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,26 +3,39 @@ "ok": true, "generated_at": "2026-06-15", "skill_dir": ".", - "commit": "038891715d7843daf7960c2ec61bf7d9ea91c9e4", + "commit": "effbf3f74c4ce959c7444d873d0586d884a43202", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 65, + "sample": [ + " M Makefile", + " M README.md", + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "b8858dc09a54ae5d02549e2a4ad6fb9f6c0237ebb6be7a54c61b147aaff7f592", - "source_contract_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58", + "evidence_bundle_sha256": "c3a7eae59138523528287cb2670c2c153ec3c2e27d14d09ab01d5a671cb98f49", + "source_contract_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", "output_case_count": 5, "failure_disclosure_count": 3, - "command_count": 21, + "command_count": 22, "command_executed_count": 10, "timing_observed_count": 10, "model_executed_count": 0, @@ -34,29 +47,30 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 7, - "world_class_source_blocked_count": 6, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 65 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (7/13 pass, 6 blocked)" + "world-class source checks are not all accepted (6/13 pass, 7 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "038891715d7843daf7960c2ec61bf7d9ea91c9e4", + "ready": false, + "commit": "effbf3f74c4ce959c7444d873d0586d884a43202", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 24, "missing_count": 0, "missing_paths": [], - "sha256": "b8858dc09a54ae5d02549e2a4ad6fb9f6c0237ebb6be7a54c61b147aaff7f592" + "sha256": "c3a7eae59138523528287cb2670c2c153ec3c2e27d14d09ab01d5a671cb98f49" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -138,7 +152,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "60a9a34e5e2ea2c239edb5037ce126a5e1b0fbfc3ae6fbb7d686fa411a225068" + "sha256": "d16d8e11c63928e7a6068e46f477b0717e91e8f1b4bc2966c92684bf3cbae4dd" }, { "label": "blind_review", @@ -172,22 +186,22 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 105694, - "sha256": "05b721899b8196240ba3dbdb442654d03cadecce6b162dac315d7fb238783c24" + "bytes": 106775, + "sha256": "12d664865a0a38879dc32db7f38c201cac405939476e0f52eadc4c957384564c" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 22252, - "sha256": "2b8c4b338b5c70e1643ab20f8ef42329d69dd9876f964fdaaa454f90520adddc" + "bytes": 22510, + "sha256": "73c6c2a81af9980cc1d8e2d1bb5dfb30bd8daccac48125fe71ec359dbde824dd" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "efc0d48985c1fbb06205a32b3ef40cb5e69795b39c1385953bdc16f0a58f726b" + "sha256": "d5e08cc39a3c5073a9332e990c566960ab8166cbdeb3980a8eb63413da64fe21" }, { "label": "package_verification", @@ -201,70 +215,70 @@ "path": "reports/install_simulation.json", "exists": true, "bytes": 8758, - "sha256": "46d0524a6968a1dadfd63988cd00b218323b42f4e7697fe427d8f160da72c252" + "sha256": "490e1f665580f5d9794b05bd0fcc57342dc66971776577037176f181a5c2c875" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14310, - "sha256": "8481f4d78f2ad4ff0e74936b58b643a91052a30b6d12bcbbf24e0b4291b8fb1f" + "sha256": "a4cf40478f3ad9404a9cb3ccff9f152481703b8936c33841ae898fd589b78fa0" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, "bytes": 19940, - "sha256": "b408af112c784fdc3e96ea9ecf374990b9f58a0ebfa4df18394ccd9e93dc5545" + "sha256": "933cdb0021818c0a8c5fc199af7dfa7879777ebf7959ed3a36bba52a0aed9c6c" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 20003, - "sha256": "cc886713ba5b99bdb2f740217f0979a30312aef7539d2a22af0e96ec5c3e90bf" + "bytes": 20006, + "sha256": "5407409841eb8e1174a07ea0af17ddf709155f0c548c897b82c2f75b869b333a" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, "bytes": 18865, - "sha256": "2fbecc603d8dc6382fde6c61869aaf4b56fc9d1be5afcd07b345c419dce6d55a" + "sha256": "b10e1ce0a5a17df72462ce8f0c468356f58ff88bd7ceab349c82a1fb3b6697ef" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 12410, - "sha256": "bc690472684dcdb8c77803b730857f98e6b6eb11f1e741d5fcc1aa9303b77e47" + "bytes": 12413, + "sha256": "3bce5f072d037a6c383cfd597f8530c2bc87ba31781fa90c843a9186dd2f8fea" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 23957, - "sha256": "b4640688026ef9199e96f80bd2d9971aaca3ec6d092d8ecd052a5debc0aa94c7" + "bytes": 24021, + "sha256": "d377b8d99831ce297011e4e270fc7287a53727500f5d503dd3dc4b984d26f0ad" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 15025, - "sha256": "3a916ac568377d8df7d612426f7358283eddfa2ab5fedb7ed12db0097a8b3c48" + "bytes": 15080, + "sha256": "9f141f09bf485a4299b37eb8c21ebab9f212ea130a0daea8b0d7d5bd7ab020aa" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 20969, - "sha256": "886b915da0f27fb46f4d8cebb6211ab5066a9fd44e1ec2e3206487d9a8abfff2" + "bytes": 21030, + "sha256": "04cc091b113f018072d07a1d71ad142773c415a49e633493689903e7b399b5e6" }, { "label": "world_class_claim_guard", "path": "reports/world_class_claim_guard.json", "exists": true, - "bytes": 8725, - "sha256": "ba4a3f36c2a9468bbb286b2c69910ffbba1e9a3f622ddf13e39fcb29116952f9" + "bytes": 8814, + "sha256": "7e5a2eac1020f4f58688b52d3c59b5df1c74d7bc04f4e599b48e5be4cc23d786" } ], "missing_artifacts": [], @@ -369,6 +383,11 @@ "command": "python3 scripts/yao.py world-class-claim-guard .", "evidence": "reports/world_class_claim_guard.json" }, + { + "label": "evidence consistency", + "command": "python3 scripts/yao.py evidence-consistency .", + "evidence": "reports/evidence_consistency.json" + }, { "label": "full ci", "command": "make ci-test", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 5d6484a5..2e0b0c69 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,29 +1,29 @@ # Benchmark Reproducibility Generated at: `2026-06-15` -Commit: `038891715d7843daf7960c2ec61bf7d9ea91c9e4` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `b8858dc09a54ae5d02549e2a4ad6fb9f6c0237ebb6be7a54c61b147aaff7f592` +Commit: `effbf3f74c4ce959c7444d873d0586d884a43202` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `c3a7eae59138523528287cb2670c2c153ec3c2e27d14d09ab01d5a671cb98f49` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `24` - missing artifacts: `0` -- source contract sha256: `652c3a4a1f09` +- source contract sha256: `278b4ff6c0c7` - archive sha256: `6852cf91a74d` - output cases: `5` - disclosed failure cases: `3` -- reproduction commands: `21` +- reproduction commands: `22` - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- world-class source checks: `7` pass / `13` total; `6` blocked +- world-class source checks: `6` pass / `13` total; `7` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `65` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,22 +35,23 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | -| world-class source checks are not all accepted (7/13 pass, 6 blocked) | +| world-class source checks are not all accepted (6/13 pass, 7 blocked) | ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `24` / `24` -- sha256: `b8858dc09a54ae5d02549e2a4ad6fb9f6c0237ebb6be7a54c61b147aaff7f592` +- sha256: `c3a7eae59138523528287cb2670c2c153ec3c2e27d14d09ab01d5a671cb98f49` ## Methodology Sections @@ -72,25 +73,25 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `60a9a34e5e2e` | +| output_execution | `reports/output_execution_runs.json` | present | `d16d8e11c639` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | -| trust_report | `reports/security_trust_report.json` | present | `05b721899b81` | -| python_compatibility | `reports/python_compatibility.json` | present | `2b8c4b338b5c` | -| registry_audit | `reports/registry_audit.json` | present | `efc0d48985c1` | +| trust_report | `reports/security_trust_report.json` | present | `12d664865a0a` | +| python_compatibility | `reports/python_compatibility.json` | present | `73c6c2a81af9` | +| registry_audit | `reports/registry_audit.json` | present | `d5e08cc39a3c` | | package_verification | `reports/package_verification.json` | present | `2476ae8ec9c4` | -| install_simulation | `reports/install_simulation.json` | present | `46d0524a6968` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `8481f4d78f2a` | -| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `b408af112c78` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `cc886713ba5b` | -| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `2fbecc603d8d` | -| world_class_submission_review | `reports/world_class_submission_review.json` | present | `bc690472684d` | -| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `b4640688026e` | -| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `3a916ac56837` | -| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `886b915da0f2` | -| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `ba4a3f36c2a9` | +| install_simulation | `reports/install_simulation.json` | present | `490e1f665580` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `a4cf40478f3a` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `933cdb002181` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `5407409841eb` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `b10e1ce0a5a1` | +| world_class_submission_review | `reports/world_class_submission_review.json` | present | `3bce5f072d03` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `d377b8d99831` | +| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `9f141f09bf48` | +| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `04cc091b113f` | +| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `7e5a2eac1020` | ## Reproduction Commands @@ -134,6 +135,8 @@ This report proves local benchmark reproducibility only. It keeps external provi - evidence: `reports/world_class_operator_runbook.json` - `python3 scripts/yao.py world-class-claim-guard .` - evidence: `reports/world_class_claim_guard.json` +- `python3 scripts/yao.py evidence-consistency .` + - evidence: `reports/evidence_consistency.json` - `make ci-test` - evidence: `CI target output` diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index f9fa4782..8f1ef92e 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { "target_count": 5, @@ -186,6 +186,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -226,8 +227,7 @@ "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", - "scripts/run_eval_suite.py", - "scripts/run_output_eval.py" + "scripts/run_eval_suite.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -387,7 +387,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -418,6 +418,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -494,14 +495,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -523,7 +524,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -636,7 +637,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -667,6 +668,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -743,14 +745,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -770,7 +772,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -1045,6 +1047,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1085,8 +1088,7 @@ "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", - "scripts/run_eval_suite.py", - "scripts/run_output_eval.py" + "scripts/run_eval_suite.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -1246,7 +1248,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1277,6 +1279,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1353,14 +1356,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1382,7 +1385,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -1495,7 +1498,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1526,6 +1529,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1602,14 +1606,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1629,7 +1633,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -1904,6 +1908,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1944,8 +1949,7 @@ "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", - "scripts/run_eval_suite.py", - "scripts/run_output_eval.py" + "scripts/run_eval_suite.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -2105,7 +2109,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2136,6 +2140,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -2212,14 +2217,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2241,7 +2246,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -2347,7 +2352,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2378,6 +2383,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -2454,14 +2460,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2481,7 +2487,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -2747,6 +2753,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -2787,8 +2794,7 @@ "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", - "scripts/run_eval_suite.py", - "scripts/run_output_eval.py" + "scripts/run_eval_suite.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -2948,7 +2954,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2979,6 +2985,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -3055,14 +3062,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3084,7 +3091,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -3190,7 +3197,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3221,6 +3228,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -3297,14 +3305,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3324,7 +3332,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -3590,6 +3598,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -3630,8 +3639,7 @@ "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", - "scripts/run_eval_suite.py", - "scripts/run_output_eval.py" + "scripts/run_eval_suite.py" ], "assets": [ "templates/basic_skill.md.j2", @@ -3791,7 +3799,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3822,6 +3830,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -3898,14 +3907,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3927,7 +3936,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, @@ -4037,7 +4046,7 @@ }, "file_write": { "required": true, - "script_count": 67, + "script_count": 68, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4068,6 +4077,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -4144,14 +4154,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 80, + "checked_count": 81, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 67, + "file_write_script_count": 68, "subprocess_script_count": 9, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4171,7 +4181,7 @@ ], "capability_counts": { "network": 3, - "file_write": 67, + "file_write": 68, "subprocess": 9, "interactive": 0 }, diff --git a/reports/context_budget.json b/reports/context_budget.json index 2dc9c29a..e125956e 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1308653, + "other_text_tokens": 1332754, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1309420, - "deferred_resource_tokens": 416426, + "estimated_total_text_tokens": 1333521, + "deferred_resource_tokens": 421412, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 367836, - "file_count": 106 + "estimated_tokens": 372822, + "file_count": 107 }, { "path": "references", @@ -31,8 +31,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 367836, - "file_count": 106 + "estimated_tokens": 372822, + "file_count": 107 } ], "deferred_resource_governance": { @@ -54,14 +54,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 367836, - "file_count": 106, + "estimated_tokens": 372822, + "file_count": 107, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 544, + "relevant_file_count": 548, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 135.4 diff --git a/reports/context_budget.md b/reports/context_budget.md index ec387c7c..514daf38 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 960 | 767 | 416426 | `governed` | scripts:367836 | 135.4 | - | ok | +| root | `.` | `production` | 1000 | 960 | 767 | 421412 | `governed` | scripts:372822 | 135.4 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 0e008a8f..a98d667f 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 960, "skill_body_tokens": 767, - "deferred_resource_tokens": 416426, + "deferred_resource_tokens": 421412, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 367836, - "file_count": 106 + "estimated_tokens": 372822, + "file_count": 107 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 367836, - "file_count": 106, + "estimated_tokens": 372822, + "file_count": 107, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json new file mode 100644 index 00000000..2e62acd6 --- /dev/null +++ b/reports/evidence_consistency.json @@ -0,0 +1,1648 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-15", + "skill_dir": ".", + "summary": { + "check_count": 26, + "pass_count": 26, + "warn_count": 0, + "fail_count": 0, + "decision": "consistent" + }, + "status_counts": { + "pass": 26, + "warn": 0, + "fail": 0 + }, + "checks": [ + { + "key": "required-report-artifacts", + "label": "Required report artifacts are readable", + "status": "pass", + "expected": "all required JSON reports exist and parse", + "actual": "all readable", + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json", + "reports/skill-interpretation.json", + "reports/adoption_drift_report.json", + "reports/world_class_evidence_ledger.json", + "reports/skill_os2_coverage.json", + "reports/review-studio.json" + ], + "detail": "The consistency gate can only be trusted when every source report is present and valid JSON." + }, + { + "key": "benchmark-release-lock-self-consistency", + "label": "Benchmark release lock matches git dirty state", + "status": "pass", + "expected": false, + "actual": false, + "paths": [ + "reports/benchmark_reproducibility.json" + ], + "detail": "The benchmark release lock must reflect the generation-time git dirty flag." + }, + { + "key": "overview-benchmark-commit", + "label": "overview embeds the benchmark commit", + "status": "pass", + "expected": "effbf3f74c4ce959c7444d873d0586d884a43202", + "actual": "effbf3f74c4ce959c7444d873d0586d884a43202", + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json" + ], + "detail": "Human-facing reports must point to the same benchmark release-lock commit." + }, + { + "key": "overview-benchmark-summary", + "label": "overview embeds benchmark summary fields", + "status": "pass", + "expected": { + "release_lock_ready": false, + "required_artifact_count": 24, + "missing_artifact_count": 0, + "source_contract_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", + "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 13, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, + "public_claim_ready": false, + "public_claim_blocker_count": 5 + }, + "actual": { + "release_lock_ready": false, + "required_artifact_count": 24, + "missing_artifact_count": 0, + "source_contract_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", + "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 13, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, + "public_claim_ready": false, + "public_claim_blocker_count": 5 + }, + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "overview-adoption-summary", + "label": "overview embeds adoption drift summary fields", + "status": "pass", + "expected": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "source_types": { + "manual": 1 + } + }, + "actual": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "source_types": { + "manual": 1 + } + }, + "paths": [ + "reports/adoption_drift_report.json", + "reports/skill-overview.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "overview-world-class-ledger-summary", + "label": "overview embeds world-class ledger summary fields", + "status": "pass", + "expected": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "actual": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill-overview.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "overview-world-class-readiness", + "label": "overview derives readiness from the ledger", + "status": "pass", + "expected": { + "ready": false, + "decision": "evidence-pending", + "pending_count": 4, + "accepted_count": 0, + "source_check_count": 13, + "source_pass_count": 6 + }, + "actual": { + "ready": false, + "decision": "evidence-pending", + "pending_count": 4, + "accepted_count": 0, + "source_check_count": 13, + "source_pass_count": 6 + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill-overview.json" + ], + "detail": "Readiness summaries must be derived from the evidence ledger, not hand-maintained copy." + }, + { + "key": "interpretation-benchmark-commit", + "label": "interpretation embeds the benchmark commit", + "status": "pass", + "expected": "effbf3f74c4ce959c7444d873d0586d884a43202", + "actual": "effbf3f74c4ce959c7444d873d0586d884a43202", + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-interpretation.json" + ], + "detail": "Human-facing reports must point to the same benchmark release-lock commit." + }, + { + "key": "interpretation-benchmark-summary", + "label": "interpretation embeds benchmark summary fields", + "status": "pass", + "expected": { + "release_lock_ready": false, + "required_artifact_count": 24, + "missing_artifact_count": 0, + "source_contract_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", + "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 13, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, + "public_claim_ready": false, + "public_claim_blocker_count": 5 + }, + "actual": { + "release_lock_ready": false, + "required_artifact_count": 24, + "missing_artifact_count": 0, + "source_contract_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", + "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 13, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, + "public_claim_ready": false, + "public_claim_blocker_count": 5 + }, + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-interpretation.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "interpretation-adoption-summary", + "label": "interpretation embeds adoption drift summary fields", + "status": "pass", + "expected": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "source_types": { + "manual": 1 + } + }, + "actual": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "source_types": { + "manual": 1 + } + }, + "paths": [ + "reports/adoption_drift_report.json", + "reports/skill-interpretation.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "interpretation-world-class-ledger-summary", + "label": "interpretation embeds world-class ledger summary fields", + "status": "pass", + "expected": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "actual": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill-interpretation.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "interpretation-world-class-readiness", + "label": "interpretation derives readiness from the ledger", + "status": "pass", + "expected": { + "ready": false, + "decision": "evidence-pending", + "pending_count": 4, + "accepted_count": 0, + "source_check_count": 13, + "source_pass_count": 6 + }, + "actual": { + "ready": false, + "decision": "evidence-pending", + "pending_count": 4, + "accepted_count": 0, + "source_check_count": 13, + "source_pass_count": 6 + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill-interpretation.json" + ], + "detail": "Readiness summaries must be derived from the evidence ledger, not hand-maintained copy." + }, + { + "key": "overview-interpretation-lockstep-scorecard", + "label": "Overview and interpretation share scorecard", + "status": "pass", + "expected": { + "completeness_score": { + "label": "完整度", + "score": 100, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 100, + "reasons": [ + "已生成 20 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 100, + "reasons": [ + "SKILL.md 约 356 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 已承载确定性逻辑。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 42, + "reasons": [ + "入口约 356 个词/字，references 约 16045 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本偏高，建议压缩入口或拆分 references。" + ] + } + }, + "actual": { + "completeness_score": { + "label": "完整度", + "score": 100, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 100, + "reasons": [ + "已生成 20 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 100, + "reasons": [ + "SKILL.md 约 356 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 已承载确定性逻辑。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 42, + "reasons": [ + "入口约 356 个词/字，references 约 16045 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本偏高，建议压缩入口或拆分 references。" + ] + } + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-capability-profile", + "label": "Overview and interpretation share capability_profile", + "status": "pass", + "expected": { + "archetype": "governed", + "task_family": "Execution operation", + "maturity": "governed", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "actual": { + "archetype": "governed", + "task_family": "Execution operation", + "maturity": "governed", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-principle-model", + "label": "Overview and interpretation share principle_model", + "status": "pass", + "expected": { + "nodes": [ + { + "title": "意图澄清", + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "actual": { + "nodes": [ + { + "title": "意图澄清", + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-contract-boundary", + "label": "Overview and interpretation share contract_boundary", + "status": "pass", + "expected": { + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "outputs": [ + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "actual": { + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "outputs": [ + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-quality-review", + "label": "Overview and interpretation share quality_review", + "status": "pass", + "expected": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" + ], + "gaps": [ + "上下文成本需要补强：入口约 356 个词/字，references 约 16045 个词/字。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations.", + "Prefer the smallest runnable snippet over broad framework scaffolding." + ] + }, + "prompt_quality": { + "overall_quality_score": 89.0, + "highlights": [ + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." + ] + }, + "system_model": { + "stability": { + "score": 100, + "band": "system-ready" + }, + "highlights": [ + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." + ] + } + }, + "actual": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" + ], + "gaps": [ + "上下文成本需要补强：入口约 356 个词/字，references 约 16045 个词/字。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations.", + "Prefer the smallest runnable snippet over broad framework scaffolding." + ] + }, + "prompt_quality": { + "overall_quality_score": 89.0, + "highlights": [ + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." + ] + }, + "system_model": { + "stability": { + "score": 100, + "band": "system-ready" + }, + "highlights": [ + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." + ] + } + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-risk-governance", + "label": "Overview and interpretation share risk_governance", + "status": "pass", + "expected": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 1, + "signal": "SKILL.md 约 356 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "actual": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 1, + "signal": "SKILL.md 约 356 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-world-class-readiness", + "label": "Overview and interpretation share world_class_readiness", + "status": "pass", + "expected": { + "ready": false, + "decision": "evidence-pending", + "entry_count": 4, + "pending_count": 4, + "accepted_count": 0, + "external_pending_count": 3, + "human_pending_count": 1, + "source_check_count": 13, + "source_pass_count": 6, + "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", + "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", + "entries": [ + { + "key": "provider-holdout", + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + "blocked_checks": [ + "Provider model run", + "Token usage observed" + ] + }, + { + "key": "human-adjudication", + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "category": "human", + "category_zh": "人工证据", + "category_en": "Human evidence", + "status": "pending", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + "blocked_checks": [ + "No pending decisions", + "Judgments complete" + ] + }, + { + "key": "native-permission-enforcement", + "label_zh": "原生权限", + "label_en": "Native Permission", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + "blocked_checks": [ + "Native enforcement" + ] + }, + { + "key": "native-client-telemetry", + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + "blocked_checks": [ + "External events", + "Adoption sample" + ] + } + ] + }, + "actual": { + "ready": false, + "decision": "evidence-pending", + "entry_count": 4, + "pending_count": 4, + "accepted_count": 0, + "external_pending_count": 3, + "human_pending_count": 1, + "source_check_count": 13, + "source_pass_count": 6, + "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", + "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", + "entries": [ + { + "key": "provider-holdout", + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + "blocked_checks": [ + "Provider model run", + "Token usage observed" + ] + }, + { + "key": "human-adjudication", + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "category": "human", + "category_zh": "人工证据", + "category_en": "Human evidence", + "status": "pending", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + "blocked_checks": [ + "No pending decisions", + "Judgments complete" + ] + }, + { + "key": "native-permission-enforcement", + "label_zh": "原生权限", + "label_en": "Native Permission", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + "blocked_checks": [ + "Native enforcement" + ] + }, + { + "key": "native-client-telemetry", + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + "blocked_checks": [ + "External events", + "Adoption sample" + ] + } + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-package-assets", + "label": "Overview and interpretation share package_assets", + "status": "pass", + "expected": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 34 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 107 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 29 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 215 + } + ], + "file_count": 389, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 34 + }, + { + "label": "scripts", + "value": 107 + }, + { + "label": "evals", + "value": 29 + }, + { + "label": "reports", + "value": 215 + } + ] + }, + "actual": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 34 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 107 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 29 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 215 + } + ], + "file_count": 389, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 34 + }, + { + "label": "scripts", + "value": 107 + }, + { + "label": "evals", + "value": 29 + }, + { + "label": "reports", + "value": 215 + } + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-iteration-roadmap", + "label": "Overview and interpretation share iteration_roadmap", + "status": "pass", + "expected": { + "items": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "继续补齐剩余 2 项外部/人工证据，并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + } + ] + }, + "actual": { + "items": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "继续补齐剩余 2 项外部/人工证据，并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + } + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-html-contract", + "label": "overview has a stable HTML contract", + "status": "pass", + "expected": { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": "reports/skill-overview.html", + "html_exists": true + }, + "actual": { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": "reports/skill-overview.html", + "html_exists": true + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-overview.html" + ], + "detail": "Report output paths and language defaults are part of the user-facing contract." + }, + { + "key": "interpretation-html-contract", + "label": "interpretation has a stable HTML contract", + "status": "pass", + "expected": { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": "reports/skill-interpretation.html", + "html_exists": true + }, + "actual": { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": "reports/skill-interpretation.html", + "html_exists": true + }, + "paths": [ + "reports/skill-interpretation.json", + "reports/skill-interpretation.html" + ], + "detail": "Report output paths and language defaults are part of the user-facing contract." + }, + { + "key": "coverage-world-class-boundary", + "label": "Coverage report mirrors world-class evidence boundary", + "status": "pass", + "expected": { + "world_class_evidence_pending_count": 4, + "public_world_class_ready": false + }, + "actual": { + "world_class_evidence_pending_count": 4, + "public_world_class_ready": false + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill_os2_coverage.json" + ], + "detail": "Blueprint coverage can be locally complete while public world-class evidence remains pending." + }, + { + "key": "benchmark-world-class-boundary", + "label": "Benchmark report mirrors world-class evidence boundary", + "status": "pass", + "expected": { + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 13, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, + "public_claim_ready": false + }, + "actual": { + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 13, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, + "public_claim_ready": false + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/benchmark_reproducibility.json" + ], + "detail": "Benchmark reproducibility must not overstate public claim readiness." + }, + { + "key": "review-studio-no-overclaim", + "label": "Review Studio does not overclaim pending world-class evidence", + "status": "pass", + "expected": false, + "actual": false, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/review-studio.json" + ], + "detail": "When world-class evidence is pending, Review Studio must stay in a review or warning posture." + } + ], + "artifacts": { + "json": "reports/evidence_consistency.json", + "markdown": "reports/evidence_consistency.md" + } +} diff --git a/reports/evidence_consistency.md b/reports/evidence_consistency.md new file mode 100644 index 00000000..c2e57fb2 --- /dev/null +++ b/reports/evidence_consistency.md @@ -0,0 +1,44 @@ +# Evidence Consistency + +Generated at: `2026-06-15` + +## Summary + +- decision: `consistent` +- checks: `26` +- pass: `26` +- warn: `0` +- fail: `0` + +This gate compares generated evidence reports against each other. It does not create provider, human, native-client, or permission-enforcement evidence; it only catches drift between reports that already exist. + +## Checks + +| Check | Status | Detail | Paths | +| --- | --- | --- | --- | +| Required report artifacts are readable | `pass` | The consistency gate can only be trusted when every source report is present and valid JSON. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json`, `reports/skill-interpretation.json`, `reports/adoption_drift_report.json`, `reports/world_class_evidence_ledger.json`, `reports/skill_os2_coverage.json`, `reports/review-studio.json` | +| Benchmark release lock matches git dirty state | `pass` | The benchmark release lock must reflect the generation-time git dirty flag. | `reports/benchmark_reproducibility.json` | +| overview embeds the benchmark commit | `pass` | Human-facing reports must point to the same benchmark release-lock commit. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json` | +| overview embeds benchmark summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json` | +| overview embeds adoption drift summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/adoption_drift_report.json`, `reports/skill-overview.json` | +| overview embeds world-class ledger summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/world_class_evidence_ledger.json`, `reports/skill-overview.json` | +| overview derives readiness from the ledger | `pass` | Readiness summaries must be derived from the evidence ledger, not hand-maintained copy. | `reports/world_class_evidence_ledger.json`, `reports/skill-overview.json` | +| interpretation embeds the benchmark commit | `pass` | Human-facing reports must point to the same benchmark release-lock commit. | `reports/benchmark_reproducibility.json`, `reports/skill-interpretation.json` | +| interpretation embeds benchmark summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/benchmark_reproducibility.json`, `reports/skill-interpretation.json` | +| interpretation embeds adoption drift summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/adoption_drift_report.json`, `reports/skill-interpretation.json` | +| interpretation embeds world-class ledger summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/world_class_evidence_ledger.json`, `reports/skill-interpretation.json` | +| interpretation derives readiness from the ledger | `pass` | Readiness summaries must be derived from the evidence ledger, not hand-maintained copy. | `reports/world_class_evidence_ledger.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share scorecard | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share capability_profile | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share principle_model | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share contract_boundary | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share quality_review | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share risk_governance | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share world_class_readiness | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share package_assets | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share iteration_roadmap | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| overview has a stable HTML contract | `pass` | Report output paths and language defaults are part of the user-facing contract. | `reports/skill-overview.json`, `reports/skill-overview.html` | +| interpretation has a stable HTML contract | `pass` | Report output paths and language defaults are part of the user-facing contract. | `reports/skill-interpretation.json`, `reports/skill-interpretation.html` | +| Coverage report mirrors world-class evidence boundary | `pass` | Blueprint coverage can be locally complete while public world-class evidence remains pending. | `reports/world_class_evidence_ledger.json`, `reports/skill_os2_coverage.json` | +| Benchmark report mirrors world-class evidence boundary | `pass` | Benchmark reproducibility must not overstate public claim readiness. | `reports/world_class_evidence_ledger.json`, `reports/benchmark_reproducibility.json` | +| Review Studio does not overclaim pending world-class evidence | `pass` | When world-class evidence is pending, Review Studio must stay in a review or warning posture. | `reports/world_class_evidence_ledger.json`, `reports/review-studio.json` | diff --git a/reports/install_simulation.json b/reports/install_simulation.json index f1b7e156..e57c37e7 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 603, + "archive_entry_count": 607, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index ac946c92..051ca367 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.86, + "duration_ms": 33.48, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 34.88, + "duration_ms": 31.57, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 42.96, + "duration_ms": 31.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 40.67, + "duration_ms": 31.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 41.06, + "duration_ms": 30.98, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 39.95, + "duration_ms": 31.67, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 42.05, + "duration_ms": 30.89, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 41.65, + "duration_ms": 32.74, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 43.42, + "duration_ms": 33.47, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 44.08, + "duration_ms": 33.37, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 6903b0bb..81a6772f 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 33.86 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 34.88 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 42.96 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 40.67 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 41.06 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 39.95 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 42.05 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 41.65 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 43.42 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 44.08 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 33.48 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 31.57 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 31.37 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.21 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 30.98 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 31.67 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 30.89 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 32.74 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 33.47 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 33.37 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 010042be..3a194308 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -1,11 +1,11 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "root": ".", "summary": { "target_python": "3.11", - "file_count": 171, + "file_count": 173, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -268,6 +268,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/render_evidence_consistency.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/render_intent_confidence.py", "ok": true, @@ -736,6 +742,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_evidence_consistency.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_failure_regressions.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index e5e40c4e..8fcded0e 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -1,12 +1,12 @@ # Python Compatibility -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary - decision: `pass` - target python: `3.11` -- files scanned: `171` +- files scanned: `173` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 76caad75..fa059d89 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,7 +21,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58", + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc" }, "compatibility": { @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58" + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 3db9e71b..4295bad7 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,7 +6,7 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58` +- Package SHA256: `278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713` - Archive SHA256: `6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc` - Install simulated: `True` diff --git a/reports/review-viewer.json b/reports/review-viewer.json index c5057252..90ac8808 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -513,7 +513,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 106 + "file_count": 107 }, { "path": "evals", @@ -525,10 +525,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 213 + "file_count": 215 } ], - "file_count": 386, + "file_count": 389, "folder_count": 4, "distribution": [ { @@ -553,7 +553,7 @@ }, { "label": "scripts", - "value": 106 + "value": 107 }, { "label": "evals", @@ -561,7 +561,7 @@ }, { "label": "reports", - "value": 213 + "value": 215 } ] }, @@ -683,7 +683,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 106 + "file_count": 107 }, { "path": "evals", @@ -695,7 +695,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 213 + "file_count": 215 } ], "strengths": [ @@ -1000,12 +1000,12 @@ "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "60df1cebf40b1627f6d313c233f377d0d36228a9525b70099623563c7efb7804", - "source_contract_sha256": "c0fe8976bca1ceac0b325c882003335c66487b8f152868af471f3bdd79b9f0af", + "evidence_bundle_sha256": "c3a7eae59138523528287cb2670c2c153ec3c2e27d14d09ab01d5a671cb98f49", + "source_contract_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", "output_case_count": 5, "failure_disclosure_count": 3, - "command_count": 21, + "command_count": 22, "command_executed_count": 10, "timing_observed_count": 10, "model_executed_count": 0, @@ -1022,9 +1022,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 73 + "changed_file_count": 65 }, - "commit": "5495734e7e33b19b84a932626112d783a3edf271", + "commit": "effbf3f74c4ce959c7444d873d0586d884a43202", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1108,8 +1108,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 193, - "script_count": 106, + "scanned_files": 194, + "script_count": 107, "internal_module_count": 26, "secret_findings": 0, "dependency_files": [ @@ -1118,18 +1118,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 67, + "file_write_script_count": 68, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 80, + "help_smoke_checked_count": 81, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 193, - "package_sha256": "c0fe8976bca1ceac0b325c882003335c66487b8f152868af471f3bdd79b9f0af" + "package_hash_file_count": 194, + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" }, "skill_atlas": { "skill_count": 12, @@ -1167,7 +1167,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "c0fe8976bca1ceac0b325c882003335c66487b8f152868af471f3bdd79b9f0af", + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc" }, "compatibility": { @@ -1227,7 +1227,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 603, + "archive_entry_count": 607, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1299,7 +1299,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "c0fe8976bca1ceac0b325c882003335c66487b8f152868af471f3bdd79b9f0af" + "to": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" } ] }, diff --git a/reports/review_annotations.json b/reports/review_annotations.json index 66f2b1d9..76fadc1f 100644 --- a/reports/review_annotations.json +++ b/reports/review_annotations.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, diff --git a/reports/review_waivers.json b/reports/review_waivers.json index 07eeccbc..3acbd17e 100644 --- a/reports/review_waivers.json +++ b/reports/review_waivers.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "summary": { "waiver_count": 0, "active_count": 0, @@ -52,7 +52,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-15 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { diff --git a/reports/review_waivers.md b/reports/review_waivers.md index dbc942d1..2fcb0546 100644 --- a/reports/review_waivers.md +++ b/reports/review_waivers.md @@ -35,7 +35,7 @@ - waiver allowed: `true` - risk: review pending 5; model-executed 0; output failures 0 - evidence: `reports/output_review_adjudication.md` -- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md` +- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-15 --evidence reports/output_review_adjudication.md` - world-class boundary: Does not count as provider, human, or public world-class completion evidence. #### Required Review diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index a7b32de5..8566c1ab 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 193, - "script_count": 106, + "scanned_files": 194, + "script_count": 107, "internal_module_count": 26, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 67, + "file_write_script_count": 68, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 80, + "help_smoke_checked_count": 81, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 193, - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58" + "package_hash_file_count": 194, + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" }, "failures": [], "warnings": [], @@ -574,6 +574,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_evidence_consistency.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a cross-report evidence consistency gate for generated Skill OS reports.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_intent_confidence.py", "interface": "cli", @@ -1561,9 +1575,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 80, - "checked_count": 80, - "passed_count": 80, + "candidate_count": 81, + "checked_count": 81, + "passed_count": 81, "failed_count": 0, "skipped_count": 26, "failed_scripts": [], @@ -1938,6 +1952,16 @@ "stdout_excerpt": "usage: render_eval_dashboard.py [-h] [--eval-dir EVAL_DIR]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_evidence_consistency.py", + "command": "python3 scripts/render_evidence_consistency.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_evidence_consistency.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [", + "stderr_excerpt": "" + }, { "path": "scripts/render_intent_confidence.py", "command": "python3 scripts/render_intent_confidence.py --help", @@ -2557,6 +2581,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 75e8eaca..b4adf6c7 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `193` -- Scripts: `106` +- Scanned files: `194` +- Scripts: `107` - Internal script modules: `26` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `67` +- File-write scripts: `68` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `80` +- CLI help smoke checked: `81` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `193` -- Package SHA256: `652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58` +- Package hash files: `194` +- Package SHA256: `278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `80` -- Passed scripts: `80` +- Checked scripts: `81` +- Passed scripts: `81` - Failed scripts: `none` ## Script Surface @@ -100,6 +100,7 @@ | scripts/render_context_reports.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_description_drift_history.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_eval_dashboard.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_evidence_consistency.py | cli | True | True | True | False | False | True | False | Renders a cross-report evidence consistency gate for generated Skill OS reports. | | scripts/render_intent_confidence.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_intent_dialogue.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_iteration_directions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-interpretation.html b/reports/skill-interpretation.html index 8585c71f..6e8bccf2 100644 --- a/reports/skill-interpretation.html +++ b/reports/skill-interpretation.html @@ -919,7 +919,7 @@ -

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks7 / 13通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks6 / 13通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
@@ -930,7 +930,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布Asset Distribution303项303 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
+
资产分布Asset Distribution389项389 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index 47cfabfc..cb3da9c0 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -412,7 +412,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 7, + "source_pass_count": 6, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -471,7 +471,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -512,7 +513,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 106 + "file_count": 107 }, { "path": "evals", @@ -524,10 +525,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 130 + "file_count": 215 } ], - "file_count": 303, + "file_count": 389, "folder_count": 4, "distribution": [ { @@ -552,7 +553,7 @@ }, { "label": "scripts", - "value": 106 + "value": 107 }, { "label": "evals", @@ -560,7 +561,7 @@ }, { "label": "reports", - "value": 130 + "value": 215 } ] }, @@ -686,7 +687,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 106 + "file_count": 107 }, { "path": "evals", @@ -698,7 +699,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 130 + "file_count": 215 } ], "strengths": [ @@ -999,16 +1000,16 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "b8858dc09a54ae5d02549e2a4ad6fb9f6c0237ebb6be7a54c61b147aaff7f592", - "source_contract_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58", + "evidence_bundle_sha256": "c3a7eae59138523528287cb2670c2c153ec3c2e27d14d09ab01d5a671cb98f49", + "source_contract_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", "output_case_count": 5, "failure_disclosure_count": 3, - "command_count": 21, + "command_count": 22, "command_executed_count": 10, "timing_observed_count": 10, "model_executed_count": 0, @@ -1020,14 +1021,14 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 7, - "world_class_source_blocked_count": 6, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 65 }, - "commit": "038891715d7843daf7960c2ec61bf7d9ea91c9e4", + "commit": "effbf3f74c4ce959c7444d873d0586d884a43202", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1111,8 +1112,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 193, - "script_count": 106, + "scanned_files": 194, + "script_count": 107, "internal_module_count": 26, "secret_findings": 0, "dependency_files": [ @@ -1121,18 +1122,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 67, + "file_write_script_count": 68, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 80, + "help_smoke_checked_count": 81, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 193, - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58" + "package_hash_file_count": 194, + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" }, "skill_atlas": { "skill_count": 12, @@ -1170,7 +1171,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58", + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc" }, "compatibility": { @@ -1230,7 +1231,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 603, + "archive_entry_count": 607, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1302,7 +1303,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58" + "to": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" } ] }, @@ -1319,14 +1320,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1335,7 +1336,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1557,7 +1558,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1609,8 +1610,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 7, - "source_blocked_count": 6, + "source_pass_count": 6, + "source_blocked_count": 7, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -1946,7 +1947,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1983,7 +1984,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -2004,8 +2005,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2021,8 +2022,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/skill-overview.html b/reports/skill-overview.html index e4434928..4936f8cf 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -919,7 +919,7 @@
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder
-

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks7 / 13通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks6 / 13通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks
  • 提供商实跑Provider model run
  • Token 用量Token usage observed
人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
@@ -930,7 +930,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布Asset Distribution303项303 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
+
资产分布Asset Distribution389项389 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 2faf297d..f62df893 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -411,7 +411,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 7, + "source_pass_count": 6, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -470,7 +470,8 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events" + "External events", + "Adoption sample" ] } ] @@ -511,7 +512,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 106 + "file_count": 107 }, { "path": "evals", @@ -523,10 +524,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 130 + "file_count": 215 } ], - "file_count": 303, + "file_count": 389, "folder_count": 4, "distribution": [ { @@ -551,7 +552,7 @@ }, { "label": "scripts", - "value": 106 + "value": 107 }, { "label": "evals", @@ -559,7 +560,7 @@ }, { "label": "reports", - "value": 130 + "value": 215 } ] }, @@ -681,7 +682,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 106 + "file_count": 107 }, { "path": "evals", @@ -693,7 +694,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 130 + "file_count": 215 } ], "strengths": [ @@ -994,16 +995,16 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "b8858dc09a54ae5d02549e2a4ad6fb9f6c0237ebb6be7a54c61b147aaff7f592", - "source_contract_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58", + "evidence_bundle_sha256": "c3a7eae59138523528287cb2670c2c153ec3c2e27d14d09ab01d5a671cb98f49", + "source_contract_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", "output_case_count": 5, "failure_disclosure_count": 3, - "command_count": 21, + "command_count": 22, "command_executed_count": 10, "timing_observed_count": 10, "model_executed_count": 0, @@ -1015,14 +1016,14 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 7, - "world_class_source_blocked_count": 6, + "world_class_source_pass_count": 6, + "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 65 }, - "commit": "038891715d7843daf7960c2ec61bf7d9ea91c9e4", + "commit": "effbf3f74c4ce959c7444d873d0586d884a43202", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1106,8 +1107,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 193, - "script_count": 106, + "scanned_files": 194, + "script_count": 107, "internal_module_count": 26, "secret_findings": 0, "dependency_files": [ @@ -1116,18 +1117,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 67, + "file_write_script_count": 68, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 80, + "help_smoke_checked_count": 81, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 193, - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58" + "package_hash_file_count": 194, + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" }, "skill_atlas": { "skill_count": 12, @@ -1165,7 +1166,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58", + "package_sha256": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc" }, "compatibility": { @@ -1225,7 +1226,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 603, + "archive_entry_count": 607, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1297,7 +1298,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58" + "to": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" } ] }, @@ -1314,14 +1315,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1330,7 +1331,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {}, "source_types": { @@ -1552,7 +1553,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1604,8 +1605,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 7, - "source_blocked_count": 6, + "source_pass_count": 6, + "source_blocked_count": 7, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -1941,7 +1942,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1978,7 +1979,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -1999,8 +2000,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -2016,8 +2017,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 8ab3b23b..7ec4ba23 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -102,6 +102,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index dd2b354c..3b95eab3 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 106; help failures 0", + "current": "secrets 0; scripts 107; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 883c409f..db2fca16 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,7 +23,7 @@ Generated at: `2026-06-13` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 24; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 106; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 107; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index c5dde3bd..4cb1f40a 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -1,13 +1,13 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { - "item_count": 20, + "item_count": 21, "module_count": 8, - "recommended_pr_count": 12, - "pass_count": 20, + "recommended_pr_count": 13, + "pass_count": 21, "warn_count": 0, "missing_count": 0, "extension_track_count": 2, @@ -21,7 +21,7 @@ "decision": "local-blueprint-covered-evidence-pending" }, "status_counts": { - "pass": 20, + "pass": 21, "warn": 0, "missing": 0 }, @@ -131,7 +131,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "106 scripts; secrets 0; help failures 0", + "current": "107 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -607,6 +607,31 @@ } ], "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "evidence-consistency", + "category": "recommended-pr", + "label": "Evidence Consistency", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "26 consistency checks", + "command": "make ci-test", + "test": "tests/verify_evidence_consistency.py", + "evidence": [ + { + "path": "scripts/render_evidence_consistency.py", + "exists": true + }, + { + "path": "reports/evidence_consistency.json", + "exists": true + }, + { + "path": "tests/verify_evidence_consistency.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." } ], "reference_extension_tracks": [ @@ -718,7 +743,7 @@ "source_blueprint": { "title": "Skill Overview / Skill OS 2.0 upgrade plan", "core_module_count": 8, - "recommended_pr_count": 12, + "recommended_pr_count": 13, "reference_extension_count": 2, "reference_extensions": [ "Skill interpretation report", diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 9e5621e4..a3023eec 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -1,13 +1,13 @@ # Skill OS 2.0 Blueprint Coverage -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary - decision: `local-blueprint-covered-evidence-pending` - local blueprint ready: `true` - public world-class ready: `false` -- pass: `20` / `20` +- pass: `21` / `21` - missing: `0` - warn: `0` - reference extensions: `2` @@ -26,7 +26,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 106 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 107 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | | Registry Distribution | `pass` | archive entries 586; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | @@ -48,6 +48,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Registry Package Format | `pass` | registry ok True | `make ci-test` | `tests/verify_registry_audit.py` | | Review Studio 2.0 | `pass` | 16 review gates | `make ci-test` | `tests/verify_review_studio.py` | | Migration V2 Docs | `pass` | migration guide present | `make ci-test` | `docs review` | +| Evidence Consistency | `pass` | 26 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | ## Reference Extension Tracks @@ -207,6 +208,13 @@ These extension tracks come from the user-supplied 2.0 reference plan. They are - existing evidence: `docs/migration-v2.md`, `reports/skill-os-2-review.md`, `README.md` - next action: Keep this item covered as the implementation evolves. +### Evidence Consistency + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `scripts/render_evidence_consistency.py`, `reports/evidence_consistency.json`, `tests/verify_evidence_consistency.py` +- next action: Keep this item covered as the implementation evolves. + ### Skill Interpretation Report - objective: User-facing deep interpretation report explains use cases, triggers, inputs, outputs, workflow, principles, boundaries, quality gates, examples, and next iterations. diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index d308ad2b..6bce04fc 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -75,7 +75,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "652c3a4a1f091d875bfacdd44da9a91bc02fec90af3fe76cf3a71a6dc80aba58" + "to": "278b4ff6c0c7e8406dfa62e5813013532af58e35d12b22f8bef25d59ac544713" } ] }, diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index 648321de..ad0e3b99 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -1,12 +1,12 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 76, + "claim_surface_count": 77, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -123,6 +123,10 @@ "path": "reports/description_optimization_suite.md", "violation_count": 0 }, + { + "path": "reports/evidence_consistency.md", + "violation_count": 0 + }, { "path": "reports/family_summary.md", "violation_count": 0 diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index 9bb78630..f5142e21 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -1,13 +1,13 @@ # World-Class Claim Guard -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary - decision: `claim-guard-pass-evidence-pending` - ledger ready to claim world-class: `false` - ledger pending evidence: `4` -- claim surfaces scanned: `76` +- claim surfaces scanned: `77` - violations: `0` - overclaim guard active: `true` diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index ed002e84..c773fa5d 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { "schema_present": true, @@ -307,7 +307,7 @@ "source_accepted": false, "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index 62c2d7ed..bc5fd179 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -1,6 +1,6 @@ # World-Class Evidence Intake -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index 6637eb6e..e245338f 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { "ledger_entry_count": 4, @@ -14,8 +14,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 7, - "source_blocked_count": 6, + "source_pass_count": 6, + "source_blocked_count": 7, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -351,7 +351,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -388,7 +388,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -409,8 +409,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -426,8 +426,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index 8ec440ff..1bc98d0c 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -1,6 +1,6 @@ # World-Class Evidence Ledger -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary @@ -8,8 +8,8 @@ Generated at: `2026-06-13` - ready to claim world-class: `false` - entries: `4` - source accepted: `0` -- source checks: `7` pass / `13` total -- source blocked: `6` +- source checks: `6` pass / `13` total +- source blocked: `7` - accepted: `0` - pending: `4` - human pending: `1` @@ -29,7 +29,7 @@ This ledger records the current evidence state. It requires both passing source | `provider-holdout` | `pending` | `missing` | `external` | model-executed 0; token-observed 0 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | `human-adjudication` | `pending` | `missing` | `human` | 0/5 decisions; pending 5 | Record real A/B choices in the decision template, then regenerate adjudication. | | `native-permission-enforcement` | `pending` | `missing` | `external` | native-enforced targets 0; installer-enforced targets 4 | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | -| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 1 | Install a real client against the native host and import production metadata-only events. | +| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. | ## Provider Holdout @@ -167,8 +167,8 @@ This ledger records the current evidence state. It requires both passing source - objective: Import production metadata-only events from a real external client into the local drift loop. - source status: `external_required` -- observed state: `{"external_source_events": 0, "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` -- source checks: `2` pass / `3` total +- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` +- source checks: `1` pass / `3` total - submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_counts_as_completion": false}` ### Provenance Requirements @@ -192,7 +192,7 @@ This ledger records the current evidence state. It requires both passing source | Check | Current | Expected | Status | | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | -| Adoption sample | `1` | `>0` | `pass` | +| Adoption sample | `0` | `>0` | `blocked` | | Raw content blocked | `False` | `false` | `pass` | ### Completion Assertions diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index 236c40fe..20b0d4e5 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { "audit_decision": "continue-iteration", @@ -141,7 +141,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -299,7 +299,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index c0db846d..6af4cf3c 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -1,6 +1,6 @@ # World-Class Evidence Plan -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary @@ -22,7 +22,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It | `provider-holdout` | `external_required` | `external` | operator with provider credentials | model-executed 0; token-observed 0 | | `human-adjudication` | `human_required` | `human` | human reviewer | 0/5 decisions; pending 5 | | `native-permission-enforcement` | `external_required` | `external` | target client or installer integrator | native-enforced targets 0; installer-enforced targets 4 | -| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 1 | +| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 0 | ## Provider Holdout diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html index e239453a..be9f2679 100644 --- a/reports/world_class_operator_runbook.html +++ b/reports/world_class_operator_runbook.html @@ -57,7 +57,7 @@ Evidence Operations

World-Class Operator Runbook

A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

-
Pending4
Awaiting4
Ready0
Source7/13
Blocked6
Invalid0
+
Pending4
Awaiting4
Ready0
Source6/13
Blocked7
Invalid0

Fast Path

  1. Run the real external or human work for one evidence item.
  2. Generate and fill the matching submission draft.
  3. Validate intake and inspect the submission review queue.
  4. Refresh the ledger and run the claim guard before making any completion claim.

Evidence Items

@@ -120,7 +120,7 @@
Owner
Browser/Chrome/IDE/provider client integrator
Ledger
pending
-
Blocked
1
+
Blocked
2
Submission
evidence/world_class/submissions/native-client-telemetry.json

Source Runbook

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
@@ -130,8 +130,8 @@

Success Checks

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

Privacy

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
-

Next Source Actions

  • Import at least one metadata-only event from a real client.
-

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 1 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+

Next Source Actions

  • Import at least one metadata-only event from a real client.
  • Telemetry must include adoption outcome evidence.
+

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.

Boundary

  • Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.
  • Valid intake means ready for submission review; ledger review still requires passing source evidence.
  • The world-class ledger and claim guard remain the source of truth.
diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index eb0e1857..dfb90499 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { "evidence_item_count": 4, @@ -12,8 +12,8 @@ "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 7, - "source_blocked_count": 6, + "source_pass_count": 6, + "source_blocked_count": 7, "ready_to_claim_world_class": false, "runbook_counts_as_completion": false, "decision": "collect-evidence" @@ -394,7 +394,7 @@ "review_state": "awaiting-submission", "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", - "current": "external source events 0; adoption samples 1", + "current": "external source events 0; adoption samples 0", "execution_runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -442,7 +442,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -463,8 +463,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -479,9 +479,10 @@ "next_action": "Telemetry must stay metadata-only." } ], - "blocked_source_check_count": 1, + "blocked_source_check_count": 2, "next_source_actions": [ - "Import at least one metadata-only event from a real client." + "Import at least one metadata-only event from a real client.", + "Telemetry must include adoption outcome evidence." ], "submission_state": { "status": "missing", diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index bef6278b..3abb08c3 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -1,6 +1,6 @@ # World-Class Operator Runbook -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary @@ -28,7 +28,7 @@ This runbook coordinates evidence collection only. It does not accept submission | `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials. | operator with provider credentials | | `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Record a reviewer choice for every pair. | human reviewer | | `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Collect real target-client or external runtime guard proof. | target client or installer integrator | -| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | +| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | ## Provider Holdout @@ -239,7 +239,7 @@ This runbook coordinates evidence collection only. It does not accept submission - objective: Import production metadata-only events from a real external client into the local drift loop. - blocking reason: No evidence packet has been submitted for review. -- blocked source checks: `1` +- blocked source checks: `2` - submission: `evidence/world_class/submissions/native-client-telemetry.json` - template: `evidence/world_class/templates/native-client-telemetry.intake.json` @@ -292,13 +292,14 @@ This runbook coordinates evidence collection only. It does not accept submission ### Next Source Actions - Import at least one metadata-only event from a real client. +- Telemetry must include adoption outcome evidence. ### Source Evidence Snapshot | Check | Current | Expected | Status | Next action | | --- | --- | --- | --- | --- | | External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | -| Adoption sample | `1` | `>0` | `pass` | Telemetry must include adoption outcome evidence. | +| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | | Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | ## Boundary diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index cb10f86b..5b79d650 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-15", "skill_dir": ".", "summary": { "review_item_count": 4, @@ -13,8 +13,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 7, - "source_blocked_count": 6, + "source_pass_count": 6, + "source_blocked_count": 7, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "awaiting-submissions" @@ -265,7 +265,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 1, + "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -286,8 +286,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 1, - "status": "pass", + "actual": 0, + "status": "blocked", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -303,8 +303,8 @@ } ], "source_check_count": 3, - "source_pass_count": 2, - "source_blocked_count": 1, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 7fe1e470..a896c218 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -1,6 +1,6 @@ # World-Class Submission Review -Generated at: `2026-06-13` +Generated at: `2026-06-15` ## Summary @@ -138,7 +138,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Source Checks - External events: 0 / >0 => blocked -- Adoption sample: 1 / >0 => pass +- Adoption sample: 0 / >0 => blocked - Raw content blocked: False / false => pass #### Completion Assertions diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 5d62f0b9..254605f2 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -63,6 +63,7 @@ DEFAULT_TARGETS = [ "world-class-runbook-check", "world-class-claim-guard-check", "benchmark-reproducibility-check", + "evidence-consistency-check", "feedback-check", "adaptation-safety-check", "adoption-drift-check", diff --git a/scripts/render_benchmark_reproducibility.py b/scripts/render_benchmark_reproducibility.py index f96ca56d..b5a9bca9 100644 --- a/scripts/render_benchmark_reproducibility.py +++ b/scripts/render_benchmark_reproducibility.py @@ -149,6 +149,11 @@ REPRODUCTION_COMMANDS = [ "command": "python3 scripts/yao.py world-class-claim-guard .", "evidence": "reports/world_class_claim_guard.json", }, + { + "label": "evidence consistency", + "command": "python3 scripts/yao.py evidence-consistency .", + "evidence": "reports/evidence_consistency.json", + }, { "label": "full ci", "command": "make ci-test", diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py new file mode 100644 index 00000000..2ddcc5ab --- /dev/null +++ b/scripts/render_evidence_consistency.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a cross-report evidence consistency gate for generated Skill OS reports." +REQUIRED_REPORTS = { + "benchmark": "reports/benchmark_reproducibility.json", + "overview": "reports/skill-overview.json", + "interpretation": "reports/skill-interpretation.json", + "adoption": "reports/adoption_drift_report.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json", + "skill_os2_coverage": "reports/skill_os2_coverage.json", + "review_studio": "reports/review-studio.json", +} +BENCHMARK_SUMMARY_KEYS = [ + "release_lock_ready", + "required_artifact_count", + "missing_artifact_count", + "source_contract_sha256", + "archive_sha256", + "world_class_ledger_pending_count", + "world_class_source_check_count", + "world_class_source_pass_count", + "world_class_source_blocked_count", + "public_claim_ready", + "public_claim_blocker_count", +] +ADOPTION_SUMMARY_KEYS = [ + "event_count", + "adoption_sample_count", + "activation_count", + "accepted_count", + "adoption_rate", + "risk_band", + "event_types", + "source_types", +] +LEDGER_SUMMARY_KEYS = [ + "ledger_entry_count", + "accepted_count", + "pending_count", + "human_pending_count", + "external_pending_count", + "source_check_count", + "source_pass_count", + "source_blocked_count", + "ready_to_claim_world_class", + "decision", +] +LOCKSTEP_SECTIONS = [ + "scorecard", + "capability_profile", + "principle_model", + "contract_boundary", + "quality_review", + "risk_governance", + "world_class_readiness", + "package_assets", + "iteration_roadmap", +] + + +def load_json(path: Path) -> tuple[dict[str, Any], str | None]: + if not path.exists(): + return {}, "missing" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return {}, f"invalid-json: {exc}" + if not isinstance(payload, dict): + return {}, "json-root-not-object" + return payload, None + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def nested(payload: dict[str, Any], path: list[str], default: Any = None) -> Any: + current: Any = payload + for key in path: + if not isinstance(current, dict) or key not in current: + return default + current = current[key] + return current + + +def add_check( + checks: list[dict[str, Any]], + *, + key: str, + label: str, + status: str, + expected: Any, + actual: Any, + paths: list[str], + detail: str, +) -> None: + checks.append( + { + "key": key, + "label": label, + "status": status, + "expected": expected, + "actual": actual, + "paths": paths, + "detail": detail, + } + ) + + +def compare_values( + checks: list[dict[str, Any]], + *, + key: str, + label: str, + expected: Any, + actual: Any, + paths: list[str], + detail: str, +) -> None: + add_check( + checks, + key=key, + label=label, + status="pass" if expected == actual else "fail", + expected=expected, + actual=actual, + paths=paths, + detail=detail, + ) + + +def compare_summary_keys( + checks: list[dict[str, Any]], + *, + key_prefix: str, + label: str, + source_summary: dict[str, Any], + embedded_summary: dict[str, Any], + keys: list[str], + paths: list[str], +) -> None: + expected = {key: source_summary.get(key) for key in keys} + actual = {key: embedded_summary.get(key) for key in keys} + compare_values( + checks, + key=key_prefix, + label=label, + expected=expected, + actual=actual, + paths=paths, + detail="Selected summary fields must match exactly across generated reports.", + ) + + +def report_contract(payload: dict[str, Any]) -> dict[str, Any]: + contract = payload.get("report_contract") + return contract if isinstance(contract, dict) else {} + + +def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: + reports: dict[str, dict[str, Any]] = {} + checks: list[dict[str, Any]] = [] + load_failures: dict[str, str] = {} + for name, relative in REQUIRED_REPORTS.items(): + payload, failure = load_json(skill_dir / relative) + reports[name] = payload + if failure: + load_failures[relative] = failure + add_check( + checks, + key="required-report-artifacts", + label="Required report artifacts are readable", + status="pass" if not load_failures else "fail", + expected="all required JSON reports exist and parse", + actual=load_failures or "all readable", + paths=list(REQUIRED_REPORTS.values()), + detail="The consistency gate can only be trusted when every source report is present and valid JSON.", + ) + + benchmark = reports["benchmark"] + overview = reports["overview"] + interpretation = reports["interpretation"] + adoption = reports["adoption"] + ledger = reports["world_class_ledger"] + coverage = reports["skill_os2_coverage"] + review_studio = reports["review_studio"] + + benchmark_summary = nested(benchmark, ["summary"], {}) + adoption_summary = nested(adoption, ["summary"], {}) + ledger_summary = nested(ledger, ["summary"], {}) + coverage_summary = nested(coverage, ["summary"], {}) + studio_summary = nested(review_studio, ["summary"], {}) + if isinstance(benchmark_summary, dict): + compare_values( + checks, + key="benchmark-release-lock-self-consistency", + label="Benchmark release lock matches git dirty state", + expected=not bool(nested(benchmark, ["git_status", "dirty"], True)), + actual=benchmark_summary.get("release_lock_ready"), + paths=[REQUIRED_REPORTS["benchmark"]], + detail="The benchmark release lock must reflect the generation-time git dirty flag.", + ) + for report_key, payload in [("overview", overview), ("interpretation", interpretation)]: + embedded_benchmark = nested(payload, ["benchmark_reproducibility"], {}) + compare_values( + checks, + key=f"{report_key}-benchmark-commit", + label=f"{report_key} embeds the benchmark commit", + expected=benchmark.get("commit"), + actual=nested(embedded_benchmark, ["commit"]), + paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS[report_key]], + detail="Human-facing reports must point to the same benchmark release-lock commit.", + ) + compare_summary_keys( + checks, + key_prefix=f"{report_key}-benchmark-summary", + label=f"{report_key} embeds benchmark summary fields", + source_summary=benchmark_summary if isinstance(benchmark_summary, dict) else {}, + embedded_summary=nested(embedded_benchmark, ["summary"], {}), + keys=BENCHMARK_SUMMARY_KEYS, + paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS[report_key]], + ) + compare_summary_keys( + checks, + key_prefix=f"{report_key}-adoption-summary", + label=f"{report_key} embeds adoption drift summary fields", + source_summary=adoption_summary if isinstance(adoption_summary, dict) else {}, + embedded_summary=nested(payload, ["adoption_drift", "summary"], {}), + keys=ADOPTION_SUMMARY_KEYS, + paths=[REQUIRED_REPORTS["adoption"], REQUIRED_REPORTS[report_key]], + ) + compare_summary_keys( + checks, + key_prefix=f"{report_key}-world-class-ledger-summary", + label=f"{report_key} embeds world-class ledger summary fields", + source_summary=ledger_summary if isinstance(ledger_summary, dict) else {}, + embedded_summary=nested(payload, ["world_class_evidence_ledger", "summary"], {}), + keys=LEDGER_SUMMARY_KEYS, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS[report_key]], + ) + readiness_expected = { + "ready": ledger_summary.get("ready_to_claim_world_class") if isinstance(ledger_summary, dict) else None, + "decision": ledger_summary.get("decision") if isinstance(ledger_summary, dict) else None, + "pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None, + "accepted_count": ledger_summary.get("accepted_count") if isinstance(ledger_summary, dict) else None, + "source_check_count": ledger_summary.get("source_check_count") if isinstance(ledger_summary, dict) else None, + "source_pass_count": ledger_summary.get("source_pass_count") if isinstance(ledger_summary, dict) else None, + } + readiness = nested(payload, ["world_class_readiness"], {}) + readiness_actual = {key: readiness.get(key) if isinstance(readiness, dict) else None for key in readiness_expected} + compare_values( + checks, + key=f"{report_key}-world-class-readiness", + label=f"{report_key} derives readiness from the ledger", + expected=readiness_expected, + actual=readiness_actual, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS[report_key]], + detail="Readiness summaries must be derived from the evidence ledger, not hand-maintained copy.", + ) + + for section in LOCKSTEP_SECTIONS: + compare_values( + checks, + key=f"overview-interpretation-lockstep-{section.replace('_', '-')}", + label=f"Overview and interpretation share {section}", + expected=overview.get(section), + actual=interpretation.get(section), + paths=[REQUIRED_REPORTS["overview"], REQUIRED_REPORTS["interpretation"]], + detail="The first-class interpretation report must stay in lockstep with the canonical overview model.", + ) + + for report_key, expected_html in [ + ("overview", "reports/skill-overview.html"), + ("interpretation", "reports/skill-interpretation.html"), + ]: + contract = report_contract(reports[report_key]) + expected = { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": expected_html, + "html_exists": True, + } + actual = { + "schema_version": contract.get("schema_version"), + "default_language": contract.get("default_language"), + "layout": contract.get("layout"), + "html_report": contract.get("html_report"), + "html_exists": (skill_dir / expected_html).exists(), + } + compare_values( + checks, + key=f"{report_key}-html-contract", + label=f"{report_key} has a stable HTML contract", + expected=expected, + actual=actual, + paths=[REQUIRED_REPORTS[report_key], expected_html], + detail="Report output paths and language defaults are part of the user-facing contract.", + ) + + if isinstance(ledger_summary, dict): + expected_boundary = { + "world_class_evidence_pending_count": ledger_summary.get("pending_count"), + "public_world_class_ready": ledger_summary.get("ready_to_claim_world_class"), + } + actual_boundary = { + "world_class_evidence_pending_count": coverage_summary.get("world_class_evidence_pending_count") + if isinstance(coverage_summary, dict) + else None, + "public_world_class_ready": coverage_summary.get("public_world_class_ready") + if isinstance(coverage_summary, dict) + else None, + } + compare_values( + checks, + key="coverage-world-class-boundary", + label="Coverage report mirrors world-class evidence boundary", + expected=expected_boundary, + actual=actual_boundary, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["skill_os2_coverage"]], + detail="Blueprint coverage can be locally complete while public world-class evidence remains pending.", + ) + benchmark_boundary = { + "world_class_ledger_pending_count": ledger_summary.get("pending_count"), + "world_class_source_check_count": ledger_summary.get("source_check_count"), + "world_class_source_pass_count": ledger_summary.get("source_pass_count"), + "world_class_source_blocked_count": ledger_summary.get("source_blocked_count"), + "public_claim_ready": ledger_summary.get("ready_to_claim_world_class"), + } + actual_benchmark_boundary = { + key: benchmark_summary.get(key) if isinstance(benchmark_summary, dict) else None + for key in benchmark_boundary + } + compare_values( + checks, + key="benchmark-world-class-boundary", + label="Benchmark report mirrors world-class evidence boundary", + expected=benchmark_boundary, + actual=actual_benchmark_boundary, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]], + detail="Benchmark reproducibility must not overstate public claim readiness.", + ) + + public_ready = bool(ledger_summary.get("ready_to_claim_world_class")) if isinstance(ledger_summary, dict) else False + compare_values( + checks, + key="review-studio-no-overclaim", + label="Review Studio does not overclaim pending world-class evidence", + expected=False if not public_ready else True, + actual=studio_summary.get("decision") in {"pass", "release", "ready", "world-class-ready"} + if isinstance(studio_summary, dict) + else None, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["review_studio"]], + detail="When world-class evidence is pending, Review Studio must stay in a review or warning posture.", + ) + status_counts: dict[str, int] = {"pass": 0, "warn": 0, "fail": 0} + for check in checks: + status_counts[check["status"]] = status_counts.get(check["status"], 0) + 1 + summary = { + "check_count": len(checks), + "pass_count": status_counts.get("pass", 0), + "warn_count": status_counts.get("warn", 0), + "fail_count": status_counts.get("fail", 0), + "decision": "consistent" if status_counts.get("fail", 0) == 0 else "evidence-drift-detected", + } + return { + "schema_version": "1.0", + "ok": summary["fail_count"] == 0, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": summary, + "status_counts": status_counts, + "checks": checks, + "artifacts": { + "json": "reports/evidence_consistency.json", + "markdown": "reports/evidence_consistency.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Evidence Consistency", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- checks: `{summary['check_count']}`", + f"- pass: `{summary['pass_count']}`", + f"- warn: `{summary['warn_count']}`", + f"- fail: `{summary['fail_count']}`", + "", + "This gate compares generated evidence reports against each other. It does not create provider, human, native-client, or permission-enforcement evidence; it only catches drift between reports that already exist.", + "", + "## Checks", + "", + "| Check | Status | Detail | Paths |", + "| --- | --- | --- | --- |", + ] + for check in report["checks"]: + paths = ", ".join(f"`{path}`" for path in check["paths"]) + lines.append( + "| " + + " | ".join( + [ + check["label"].replace("|", "\\|"), + f"`{check['status']}`", + check["detail"].replace("|", "\\|"), + paths.replace("|", "\\|"), + ] + ) + + " |" + ) + failures = [check for check in report["checks"] if check["status"] == "fail"] + if failures: + lines.extend(["", "## Failures", ""]) + for check in failures: + lines.extend( + [ + f"### {check['label']}", + "", + f"- key: `{check['key']}`", + f"- expected: `{json.dumps(check['expected'], ensure_ascii=False, sort_keys=True)}`", + f"- actual: `{json.dumps(check['actual'], ensure_ascii=False, sort_keys=True)}`", + "", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render cross-report evidence consistency checks.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/evidence_consistency.json") + parser.add_argument("--output-md", default="reports/evidence_consistency.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report(skill_dir, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_skill_os2_coverage.py b/scripts/render_skill_os2_coverage.py index a4611f61..80124acd 100644 --- a/scripts/render_skill_os2_coverage.py +++ b/scripts/render_skill_os2_coverage.py @@ -22,6 +22,7 @@ RECOMMENDED_PR_LABELS = { "registry-package-format": "Registry Package Format", "review-studio-2": "Review Studio 2.0", "migration-v2-docs": "Migration V2 Docs", + "evidence-consistency": "Evidence Consistency", } @@ -146,6 +147,7 @@ def build_coverage(skill_dir: Path, generated_at: str) -> dict[str, Any]: telemetry_hooks = load_json(reports / "telemetry_hook_recipes.json") benchmark = load_json(reports / "benchmark_reproducibility.json") world_class_ledger = load_json(reports / "world_class_evidence_ledger.json") + evidence_consistency = load_json(reports / "evidence_consistency.json") output_case_count = summary_value(output_quality, "case_count") output_delta = summary_value(output_quality, "delta", "n/a") @@ -283,6 +285,7 @@ def build_coverage(skill_dir: Path, generated_at: str) -> dict[str, Any]: ("registry-package-format", "registry/package.schema.json", "reports/registry_audit.json", "tests/verify_registry_audit.py", registry.get("ok") is True, f"registry ok {registry.get('ok') is True}"), ("review-studio-2", "scripts/render_review_studio.py", "reports/review-studio.html", "tests/verify_review_studio.py", review_studio.get("ok") is True and studio_gates >= 14, f"{studio_gates} review gates"), ("migration-v2-docs", "docs/migration-v2.md", "reports/skill-os-2-review.md", "README.md", (skill_dir / "docs" / "migration-v2.md").exists(), "migration guide present"), + ("evidence-consistency", "scripts/render_evidence_consistency.py", "reports/evidence_consistency.json", "tests/verify_evidence_consistency.py", evidence_consistency.get("ok") is True and summary_value(evidence_consistency, "fail_count") == 0, f"{summary_value(evidence_consistency, 'check_count')} consistency checks"), ] pr_items = [ build_item( @@ -432,7 +435,7 @@ def build_coverage(skill_dir: Path, generated_at: str) -> dict[str, Any]: "source_blueprint": { "title": "Skill Overview / Skill OS 2.0 upgrade plan", "core_module_count": 8, - "recommended_pr_count": 12, + "recommended_pr_count": 13, "reference_extension_count": len(extension_tracks), "reference_extensions": [ "Skill interpretation report", diff --git a/scripts/yao.py b/scripts/yao.py index 0ea5475e..4ee59352 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -17,6 +17,7 @@ from yao_cli_parser import build_parser as build_cli_parser from yao_cli_report_commands import ( command_artifact_design_profile, command_benchmark_reproducibility, + command_evidence_consistency, command_github_benchmark_scan, command_intent_confidence, command_intent_dialogue, @@ -186,6 +187,7 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_benchmark_reproducibility.py", [str(ROOT)]), run_script("render_skill_overview.py", [str(ROOT)]), run_script("render_skill_interpretation.py", [str(ROOT)]), + run_script("render_evidence_consistency.py", [str(ROOT)]), run_script("render_review_viewer.py", [str(ROOT)]), ] ) @@ -228,6 +230,7 @@ def command_report(args: argparse.Namespace) -> int: "skill_overview": "reports/skill-overview.json", "skill_interpretation": "reports/skill-interpretation.json", "skill_interpretation_html": "reports/skill-interpretation.html", + "evidence_consistency": "reports/evidence_consistency.json", "review_viewer": "reports/review-viewer.json", "review_viewer_html": "reports/review-viewer.html", }, diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index ab43c92c..11c5aff5 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -273,6 +273,16 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] benchmark_reproducibility_cmd.add_argument("--generated-at") benchmark_reproducibility_cmd.set_defaults(func=_handler(command_handlers, "command_benchmark_reproducibility")) + evidence_consistency_cmd = subparsers.add_parser( + "evidence-consistency", + help="Render cross-report evidence consistency checks.", + ) + evidence_consistency_cmd.add_argument("skill_dir", nargs="?", default=".") + evidence_consistency_cmd.add_argument("--output-json") + evidence_consistency_cmd.add_argument("--output-md") + evidence_consistency_cmd.add_argument("--generated-at") + evidence_consistency_cmd.set_defaults(func=_handler(command_handlers, "command_evidence_consistency")) + reference_scan_cmd = subparsers.add_parser( "reference-scan", help="Render a controlled benchmark scan report for a skill package.", diff --git a/scripts/yao_cli_report_commands.py b/scripts/yao_cli_report_commands.py index e6f945fa..11d92e1f 100644 --- a/scripts/yao_cli_report_commands.py +++ b/scripts/yao_cli_report_commands.py @@ -124,6 +124,10 @@ def command_benchmark_reproducibility(args: argparse.Namespace) -> int: return render_skill_report_command(args, "render_benchmark_reproducibility.py", generated_at=True) +def command_evidence_consistency(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_evidence_consistency.py", generated_at=True) + + def command_reference_scan(args: argparse.Namespace) -> int: cmd = [resolved_skill_dir(args)] for reference in args.external_reference: diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index e8f8bc4d..d77411fc 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -149,6 +149,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -189,8 +190,7 @@ "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", - "scripts/run_eval_suite.py", - "scripts/run_output_eval.py" + "scripts/run_eval_suite.py" ], "assets": [ "templates/basic_skill.md.j2", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index 0b4dc402..5406929d 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -58,6 +58,7 @@ "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", diff --git a/tests/verify_benchmark_reproducibility.py b/tests/verify_benchmark_reproducibility.py index 05162ceb..c1eb5aad 100644 --- a/tests/verify_benchmark_reproducibility.py +++ b/tests/verify_benchmark_reproducibility.py @@ -176,6 +176,7 @@ def main() -> None: assert any(command["command"] == "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py world-class-claim-guard ." for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py python-compat ." for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py evidence-consistency ." for command in payload["reproduction_commands"]), payload assert any("provider-backed" in item for item in payload["limitations"]), payload["limitations"] markdown = output_md.read_text(encoding="utf-8") assert "Benchmark Reproducibility" in markdown, markdown @@ -191,6 +192,7 @@ def main() -> None: assert "reports/benchmark_methodology.md" in markdown, markdown assert "reports/world_class_operator_runbook.html" in markdown, markdown assert "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" in markdown, markdown + assert "python3 scripts/yao.py evidence-consistency ." in markdown, markdown assert "make ci-test" in markdown, markdown print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py new file mode 100644 index 00000000..a862ab0b --- /dev/null +++ b/tests/verify_evidence_consistency.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_evidence_consistency.py" +TMP = ROOT / "tests" / "tmp_evidence_consistency" +REPORT_FILES = [ + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/skill-interpretation.json", + "reports/skill-interpretation.html", + "reports/adoption_drift_report.json", + "reports/world_class_evidence_ledger.json", + "reports/skill_os2_coverage.json", + "reports/review-studio.json", +] + + +def run(cmd: list[str], *, check: bool = False) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, check=check) + + +def copy_reports(dst: Path) -> None: + for relative in REPORT_FILES: + source = ROOT / relative + target = dst / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def refresh_embedded_reports() -> None: + script_names = [ + "render_benchmark_reproducibility.py", + "render_skill_os2_coverage.py", + "render_skill_overview.py", + "render_skill_interpretation.py", + ] + for script_name in script_names: + subprocess.run( + [sys.executable, str(ROOT / "scripts" / script_name), str(ROOT)], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + refresh_embedded_reports() + output_json = TMP / "evidence_consistency.json" + output_md = TMP / "evidence_consistency.md" + proc = run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-15", + ], + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["summary"]["decision"] == "consistent", payload + assert payload["summary"]["fail_count"] == 0, payload + assert payload["summary"]["check_count"] >= 26, payload + checks = {item["key"]: item for item in payload["checks"]} + assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"] + assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"] + assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"] + assert checks["review-studio-no-overclaim"]["status"] == "pass", checks["review-studio-no-overclaim"] + markdown = output_md.read_text(encoding="utf-8") + assert "Evidence Consistency" in markdown, markdown + assert "decision: `consistent`" in markdown, markdown + assert "does not create provider, human, native-client, or permission-enforcement evidence" in markdown, markdown + + drift_root = TMP / "drift-skill" + copy_reports(drift_root) + overview_path = drift_root / "reports" / "skill-overview.json" + overview = json.loads(overview_path.read_text(encoding="utf-8")) + overview["adoption_drift"]["summary"]["adoption_sample_count"] += 1 + overview_path.write_text(json.dumps(overview, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(drift_root), + "--output-json", + str(TMP / "drift.json"), + "--output-md", + str(TMP / "drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert drift_proc.returncode == 2, drift_proc.stdout + drift_payload = json.loads(drift_proc.stdout) + drift_checks = {item["key"]: item for item in drift_payload["checks"]} + assert drift_payload["ok"] is False, drift_payload + assert drift_payload["summary"]["decision"] == "evidence-drift-detected", drift_payload + assert drift_checks["overview-adoption-summary"]["status"] == "fail", drift_checks["overview-adoption-summary"] + assert drift_checks["interpretation-adoption-summary"]["status"] == "pass", drift_checks["interpretation-adoption-summary"] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_os2_coverage.py b/tests/verify_skill_os2_coverage.py index e42c570e..62203c6f 100644 --- a/tests/verify_skill_os2_coverage.py +++ b/tests/verify_skill_os2_coverage.py @@ -39,9 +39,9 @@ def main() -> None: summary = payload["summary"] assert summary["decision"] == "local-blueprint-covered-evidence-pending", summary assert summary["module_count"] == 8, summary - assert summary["recommended_pr_count"] == 12, summary - assert summary["item_count"] == 20, summary - assert summary["pass_count"] == 20, summary + assert summary["recommended_pr_count"] == 13, summary + assert summary["item_count"] == 21, summary + assert summary["pass_count"] == 21, summary assert summary["warn_count"] == 0, summary assert summary["missing_count"] == 0, summary assert summary["extension_track_count"] == 2, summary @@ -80,12 +80,14 @@ def main() -> None: "registry-package-format", "review-studio-2", "migration-v2-docs", + "evidence-consistency", }, prs assert prs["agent-skills-conformance"]["status"] == "pass", prs["agent-skills-conformance"] assert prs["benchmark-methodology"]["status"] == "pass", prs["benchmark-methodology"] + assert prs["evidence-consistency"]["status"] == "pass", prs["evidence-consistency"] assert any(entry["path"] == "reports/benchmark_reproducibility.json" and entry["exists"] for entry in prs["benchmark-methodology"]["evidence"]) assert payload["source_blueprint"]["core_module_count"] == 8, payload - assert payload["source_blueprint"]["recommended_pr_count"] == 12, payload + assert payload["source_blueprint"]["recommended_pr_count"] == 13, payload assert payload["source_blueprint"]["reference_extension_count"] == 2, payload extension_tracks = {item["key"]: item for item in payload["reference_extension_tracks"]} assert extension_tracks["skill-interpretation-report"]["status"] == "covered", extension_tracks @@ -121,6 +123,7 @@ def main() -> None: assert "extension partial: `1`" in markdown, markdown assert "## Core Modules" in markdown, markdown assert "## Recommended PR Coverage" in markdown, markdown + assert "Evidence Consistency" in markdown, markdown assert "## Reference Extension Tracks" in markdown, markdown assert "Skill Interpretation Report" in markdown, markdown assert "Adaptive Self-Iteration" in markdown, markdown diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 67cad9bd..95c50d21 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -82,7 +82,7 @@ def main() -> None: "quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage " "world-class-evidence world-class-ledger world-class-intake world-class-submission-kit " "world-class-submission-review world-class-runbook world-class-claim-guard " - "benchmark-reproducibility output-review-kit adapt-scan adapt-propose " + "benchmark-reproducibility evidence-consistency output-review-kit adapt-scan adapt-propose " "telemetry-import telemetry-emit telemetry-hooks --record-cli-telemetry" ).split() assert all(item in parser_help for item in expected_help), parser_help @@ -328,6 +328,15 @@ def main() -> None: assert benchmark_reproducibility_result["payload"]["summary"]["reproducibility_ready"] is True, benchmark_reproducibility_result assert benchmark_reproducibility_result["payload"]["summary"]["world_class_ready"] is False, benchmark_reproducibility_result + evidence_consistency_result = run( + "evidence-consistency", str(ROOT), + "--output-json", str(tmp_root / "evidence_consistency.json"), + "--output-md", str(tmp_root / "evidence_consistency.md"), + "--generated-at", "2026-06-15", + ) + assert evidence_consistency_result["ok"], evidence_consistency_result + assert evidence_consistency_result["payload"]["summary"]["decision"] == "consistent", evidence_consistency_result + quickstart_result = run( "quickstart", "--output-dir", @@ -750,6 +759,7 @@ def main() -> None: assert "world_class_evidence_intake" in report_result["payload"]["artifacts"], report_result assert "world_class_claim_guard" in report_result["payload"]["artifacts"], report_result assert "benchmark_reproducibility" in report_result["payload"]["artifacts"], report_result + assert "evidence_consistency" in report_result["payload"]["artifacts"], report_result assert "skill_os2_coverage" in report_result["payload"]["artifacts"], report_result assert report_result["payload"]["artifacts"]["skill_overview"] == "reports/skill-overview.json", report_result assert report_result["payload"]["artifacts"]["skill_interpretation"] == "reports/skill-interpretation.json", report_result
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder