diff --git a/registry/index.json b/registry/index.json index 796504b..096a10b 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "cd77d787e9e8f85993a28db9320f8ea329052c0a6772c32ac40dae5ddcf2072f" + "package_sha256": "c6d6f2a9b0e22b9ecdf1d8ff74a9d18b16d34729b063c813dc5e16e8328e66f9" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index a7a6b3d..9fb1aa0 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "cd77d787e9e8f85993a28db9320f8ea329052c0a6772c32ac40dae5ddcf2072f", - "archive_sha256": "8ae57b78e156f451221776b2642fbbb1f98ca99dc4210b3dfc659fde2a9a118d" + "package_sha256": "c6d6f2a9b0e22b9ecdf1d8ff74a9d18b16d34729b063c813dc5e16e8328e66f9", + "archive_sha256": "f8ad7d6f98bd00553984225a54ce6809611e868fb6653605dec71774a4b5c730" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8ae57b78e156f451221776b2642fbbb1f98ca99dc4210b3dfc659fde2a9a118d", + "archive_sha256": "f8ad7d6f98bd00553984225a54ce6809611e868fb6653605dec71774a4b5c730", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index a003ac1..e5068a1 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,10 +4,10 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 188, - "script_file_count": 119, + "python_file_count": 189, + "script_file_count": 120, "test_file_count": 69, - "internal_module_count": 36, + "internal_module_count": 37, "cli_script_count": 85, "command_handler_count": 68, "entrypoint_command_handler_count": 18, @@ -45,7 +45,7 @@ }, { "path": "tests/verify_world_class_evidence_intake.py", - "lines": 817, + "lines": 842, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -72,15 +72,15 @@ "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { - "path": "scripts/build_skill_atlas.py", - "lines": 730, + "path": "scripts/prepare_world_class_submission_kit.py", + "lines": 783, "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/prepare_world_class_submission_kit.py", - "lines": 724, + "path": "scripts/build_skill_atlas.py", + "lines": 730, "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." @@ -131,7 +131,7 @@ }, { "path": "tests/verify_world_class_evidence_intake.py", - "lines": 817, + "lines": 842, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -158,15 +158,15 @@ "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { - "path": "scripts/build_skill_atlas.py", - "lines": 730, + "path": "scripts/prepare_world_class_submission_kit.py", + "lines": 783, "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/prepare_world_class_submission_kit.py", - "lines": 724, + "path": "scripts/build_skill_atlas.py", + "lines": 730, "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index fda97f1..ae19ca3 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,10 +5,10 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `188` -- scripts: `119` +- python files: `189` +- scripts: `120` - tests: `69` -- internal modules: `36` +- internal modules: `37` - CLI scripts: `85` - Yao CLI command handlers: `68` - entrypoint command handlers: `18` @@ -32,12 +32,12 @@ No file-size hotspots found. | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/render_evidence_consistency.py` | `859` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | -| `tests/verify_world_class_evidence_intake.py` | `817` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `tests/verify_world_class_evidence_intake.py` | `842` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/skill_report_layout.py` | `808` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_model.py` | `800` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `tests/verify_review_studio.py` | `784` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `scripts/prepare_world_class_submission_kit.py` | `783` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/build_skill_atlas.py` | `730` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | -| `scripts/prepare_world_class_submission_kit.py` | `724` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/compile_skill.py` | `723` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/optimize_description.py` | `723` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | @@ -48,12 +48,12 @@ No file-size hotspots found. | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | `pass` | | `scripts/render_evidence_consistency.py` | `859` | `cli-script` | `pass` | -| `tests/verify_world_class_evidence_intake.py` | `817` | `test` | `pass` | +| `tests/verify_world_class_evidence_intake.py` | `842` | `test` | `pass` | | `scripts/skill_report_layout.py` | `808` | `internal-module` | `pass` | | `scripts/skill_report_model.py` | `800` | `internal-module` | `pass` | | `tests/verify_review_studio.py` | `784` | `test` | `pass` | +| `scripts/prepare_world_class_submission_kit.py` | `783` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `730` | `cli-script` | `pass` | -| `scripts/prepare_world_class_submission_kit.py` | `724` | `cli-script` | `pass` | | `scripts/compile_skill.py` | `723` | `cli-script` | `pass` | | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | diff --git a/reports/context_budget.json b/reports/context_budget.json index 6528081..ea93a64 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1030976, + "other_text_tokens": 1039436, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1031773, - "deferred_resource_tokens": 475273, + "estimated_total_text_tokens": 1040233, + "deferred_resource_tokens": 477478, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 425493, - "file_count": 119 + "estimated_tokens": 427698, + "file_count": 120 }, { "path": "references", @@ -31,8 +31,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 425493, - "file_count": 119 + "estimated_tokens": 427698, + "file_count": 120 } ], "deferred_resource_governance": { @@ -54,14 +54,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 425493, - "file_count": 119, + "estimated_tokens": 427698, + "file_count": 120, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 621, + "relevant_file_count": 625, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index dc7e0e2..2dcb5e3 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 475273 | `governed` | scripts:425493 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 477478 | `governed` | scripts:427698 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 733da78..9e37f09 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-06-16", + "generated_at": "2026-03-31", "targets": [ { "label": "root", @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 475273, + "deferred_resource_tokens": 477478, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 425493, - "file_count": 119 + "estimated_tokens": 427698, + "file_count": 120 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 425493, - "file_count": 119, + "estimated_tokens": 427698, + "file_count": 120, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 2842ba0..b608222 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 641, + "archive_entry_count": 644, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index edd81a1..b05485e 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.52, + "duration_ms": 28.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.46, + "duration_ms": 28.0, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.09, + "duration_ms": 28.27, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.44, + "duration_ms": 28.05, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.57, + "duration_ms": 29.85, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.52, + "duration_ms": 30.65, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.76, + "duration_ms": 31.13, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.81, + "duration_ms": 33.72, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.26, + "duration_ms": 31.79, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.5, + "duration_ms": 31.11, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 6a80fb5..f1dfdd0 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 27.52 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 26.46 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 26.09 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.44 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 29.57 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 29.52 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 29.76 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.81 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.26 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.5 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 28.04 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 28.0 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 28.27 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.05 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 29.85 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 30.65 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 31.13 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 33.72 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 31.79 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 31.11 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index b15c1e3..10dce28 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8ae57b78e156f451221776b2642fbbb1f98ca99dc4210b3dfc659fde2a9a118d", - "archive_entry_count": 641, + "archive_sha256": "f8ad7d6f98bd00553984225a54ce6809611e868fb6653605dec71774a4b5c730", + "archive_entry_count": 644, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 5dd139a..c88072f 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `8ae57b78e156f451221776b2642fbbb1f98ca99dc4210b3dfc659fde2a9a118d` +- Archive SHA256: `f8ad7d6f98bd00553984225a54ce6809611e868fb6653605dec71774a4b5c730` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 42aa5b6..238924f 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 191, + "file_count": 192, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -694,6 +694,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_submission_matrix.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/yao.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 0b1c48f..7300aa5 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-16` - decision: `pass` - target python: `3.11` -- files scanned: `191` +- files scanned: `192` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index c6a832c..adc798c 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "cd77d787e9e8f85993a28db9320f8ea329052c0a6772c32ac40dae5ddcf2072f", - "archive_sha256": "8ae57b78e156f451221776b2642fbbb1f98ca99dc4210b3dfc659fde2a9a118d" + "package_sha256": "c6d6f2a9b0e22b9ecdf1d8ff74a9d18b16d34729b063c813dc5e16e8328e66f9", + "archive_sha256": "f8ad7d6f98bd00553984225a54ce6809611e868fb6653605dec71774a4b5c730" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8ae57b78e156f451221776b2642fbbb1f98ca99dc4210b3dfc659fde2a9a118d", + "archive_sha256": "f8ad7d6f98bd00553984225a54ce6809611e868fb6653605dec71774a4b5c730", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "cd77d787e9e8f85993a28db9320f8ea329052c0a6772c32ac40dae5ddcf2072f" + "package_sha256": "c6d6f2a9b0e22b9ecdf1d8ff74a9d18b16d34729b063c813dc5e16e8328e66f9" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 2898a6e..f785d38 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `cd77d787e9e8f85993a28db9320f8ea329052c0a6772c32ac40dae5ddcf2072f` -- Archive SHA256: `8ae57b78e156f451221776b2642fbbb1f98ca99dc4210b3dfc659fde2a9a118d` +- Package SHA256: `c6d6f2a9b0e22b9ecdf1d8ff74a9d18b16d34729b063c813dc5e16e8328e66f9` +- Archive SHA256: `f8ad7d6f98bd00553984225a54ce6809611e868fb6653605dec71774a4b5c730` - Install simulated: `True` ## Compatibility diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 035419c..ba31324 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 207, - "script_count": 119, - "internal_module_count": 33, + "scanned_files": 208, + "script_count": 120, + "internal_module_count": 34, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 207, - "package_sha256": "cd77d787e9e8f85993a28db9320f8ea329052c0a6772c32ac40dae5ddcf2072f" + "package_hash_file_count": 208, + "package_sha256": "c6d6f2a9b0e22b9ecdf1d8ff74a9d18b16d34729b063c813dc5e16e8328e66f9" }, "failures": [], "warnings": [], @@ -1572,6 +1572,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_submission_matrix.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared by submission kit rendering to summarize draft, artifact, and source-check readiness.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao.py", "interface": "cli", @@ -1747,7 +1761,7 @@ "checked_count": 86, "passed_count": 86, "failed_count": 0, - "skipped_count": 33, + "skipped_count": 34, "failed_scripts": [], "results": [ { @@ -2708,6 +2722,10 @@ "path": "scripts/world_class_source_checks.py", "reason": "internal module" }, + { + "path": "scripts/world_class_submission_matrix.py", + "reason": "internal module" + }, { "path": "scripts/yao_cli_adaptation_commands.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index a28dfa8..25fa496 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `207` -- Scripts: `119` -- Internal script modules: `33` +- Scanned files: `208` +- Scripts: `120` +- Internal script modules: `34` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `207` -- Package SHA256: `cd77d787e9e8f85993a28db9320f8ea329052c0a6772c32ac40dae5ddcf2072f` +- Package hash files: `208` +- Package SHA256: `c6d6f2a9b0e22b9ecdf1d8ff74a9d18b16d34729b063c813dc5e16e8328e66f9` ## Failures @@ -171,6 +171,7 @@ | scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. | | scripts/world_class_source_checks.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to keep source-evidence readiness checks consistent. | +| scripts/world_class_submission_matrix.py | internal-module | True | False | False | False | False | False | False | Shared by submission kit rendering to summarize draft, artifact, and source-check readiness. | | scripts/yao.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/yao_cli_adaptation_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep adaptive scan/proposal/apply command handlers outside the thin CLI orchestrator. | | scripts/yao_cli_config.py | internal-module | True | False | False | False | False | False | False | Imported by yao.py for CLI target maps and side-effect-free shaping helpers. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index c18bfb0..0d9e9b1 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -17,6 +17,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - World-Class Evidence Plan v0 that turns the remaining provider holdout, human adjudication, native permission enforcement, and native client telemetry gaps into executable runbooks with success checks and privacy contracts. - World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps. - World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields. +- World-Class Submission Matrix v0 so the submission kit exposes a single operator-facing matrix for draft status, artifact readiness, source-check blockers, and the next action without counting matrix rows as completion evidence. - Benchmark Reproducibility v0 that turns public benchmark methodology, required artifacts, failure disclosure, and reproduction commands into machine-checkable release evidence. - Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review. - Review Studio Source Refs v0 so every non-pass review action can expose structured relative source/report links with best-effort line numbers. @@ -45,7 +46,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. - Weekly SkillOps Curator v0 so daily SkillOps opportunities, Skill Atlas portfolio signals, release-lock state, and world-class evidence gaps roll up into a proposal-only weekly maintenance queue without source-file writes. -This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, external clients can now emit one validated metadata-only event into a local spool, hook recipes now make Browser/Chrome/VS Code/wrapper/provider-adapter integration commands auditable, a Browser/Chrome Native Messaging host can receive length-prefixed metadata events and generate a local launcher plus manifest, validated external JSONL imports can bring those non-CLI client signals into the same drift loop, and Skill Atlas now consumes aggregate drift reports. World-class intake now rejects non-canonical submission filenames and nested raw content or credential-like fields before ledger review. Review Studio keeps pending human adjudication and pending world-class evidence visible as warnings instead of treating them as a clean pass. Deeper provider-native execution transforms, installed platform-native client telemetry implementations, provider-native installer integration, real provider holdout runs, real human adjudication decisions, accepted external evidence, and native runtime permission enforcement remain open. +This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, external clients can now emit one validated metadata-only event into a local spool, hook recipes now make Browser/Chrome/VS Code/wrapper/provider-adapter integration commands auditable, a Browser/Chrome Native Messaging host can receive length-prefixed metadata events and generate a local launcher plus manifest, validated external JSONL imports can bring those non-CLI client signals into the same drift loop, and Skill Atlas now consumes aggregate drift reports. World-class intake now rejects non-canonical submission filenames and nested raw content or credential-like fields before ledger review. The submission kit now gives operators a matrix that ties each pending world-class evidence key to its draft, artifact readiness, source blockers, and next action. Review Studio keeps pending human adjudication and pending world-class evidence visible as warnings instead of treating them as a clean pass. Deeper provider-native execution transforms, installed platform-native client telemetry implementations, provider-native installer integration, real provider holdout runs, real human adjudication decisions, accepted external evidence, and native runtime permission enforcement remain open. ## Coverage Matrix @@ -60,6 +61,7 @@ This is still not the final world-class state. Target-native behavior contracts | World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | | World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed | | World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename and nested raw-field rejection | v0 landed | +| World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `tests/verify_world_class_evidence_intake.py` with draft, artifact, source-check, and next-action matrix evidence | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | | Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed | @@ -119,13 +121,13 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `33` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `119` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `34` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `120` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `641` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `641` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `644` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `644` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | | Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | | Adoption Drift | `1` metadata-only review event, `1` adoption sample, adoption `100`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 5c40858..783ffd2 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -173,6 +173,7 @@ "scripts/verify_package.py", "scripts/world_class_evidence_contract.py", "scripts/world_class_source_checks.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index fd313ad..d9346ca 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 119; help failures 0", + "current": "secrets 0; scripts 120; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -285,7 +285,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 641; install failures 0; permission failures 0", + "current": "zip entries 644; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index ca9afbd..3f9e407 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,11 +23,11 @@ Generated at: `2026-06-16` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 119; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 120; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 641; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 644; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 22841e6..47ff104 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "119 scripts; secrets 0; help failures 0", + "current": "120 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -204,7 +204,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 641; install failures 0", + "current": "archive entries 644; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index b232831..ae0a1f3 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -26,9 +26,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 119 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 120 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 641; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 644; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 5f3311f..ca396ec 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "8ae57b78e156f451221776b2642fbbb1f98ca99dc4210b3dfc659fde2a9a118d" + "to": "f8ad7d6f98bd00553984225a54ce6809611e868fb6653605dec71774a4b5c730" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "cd77d787e9e8f85993a28db9320f8ea329052c0a6772c32ac40dae5ddcf2072f" + "to": "c6d6f2a9b0e22b9ecdf1d8ff74a9d18b16d34729b063c813dc5e16e8328e66f9" } ] }, diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py index fd05118..74bc8d8 100644 --- a/scripts/prepare_world_class_submission_kit.py +++ b/scripts/prepare_world_class_submission_kit.py @@ -11,6 +11,7 @@ from html_rendering import html_text from render_world_class_evidence_intake import build_intake from world_class_evidence_contract import DISALLOWED_REAL_ARTIFACTS from world_class_source_checks import build_source_checklist, summarize_source_checklist +from world_class_submission_matrix import build_evidence_matrix, summarize_evidence_matrix ROOT = Path(__file__).resolve().parent.parent @@ -296,6 +297,23 @@ def render_readme(report: dict[str, Any]) -> str: lines.append( f"| `{item['evidence_key']}` | `{item['output_path']}` | `{item['status']}` | `{item.get('prefilled_artifact_ref_count', 0)}` |" ) + lines.extend( + [ + "", + "## Evidence Matrix", + "", + "This matrix combines draft, artifact, and source-check readiness into one operator action list. Matrix rows are guidance only; they do not count as completion evidence.", + "", + "| Evidence | Stage | Draft | Artifacts | Source checks | Next action |", + "| --- | --- | --- | ---: | ---: | --- |", + ] + ) + for item in report.get("evidence_matrix", []): + lines.append( + f"| `{item['evidence_key']}` | `{item['stage']}` | `{item['draft_status']}` | " + f"`{item['artifact_ready_count']}/{item['artifact_total_count']}` | " + f"`{item['source_pass_count']}/{item['source_check_count']}` | {item['next_action']} |" + ) lines.extend(["", "## Execution Runbook", ""]) for item in report.get("evidence_items", []): must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} @@ -455,6 +473,39 @@ def render_html_source_checklist(items: list[dict[str, Any]]) -> str: ) +def render_html_matrix(items: list[dict[str, Any]]) -> str: + if not items: + return "

No evidence matrix rows were generated.

" + return "".join( + """ +
+
+ {stage} +

{key}

+
+
+
Draft
{draft}
+
Artifacts
{artifact_ready}/{artifact_total} ready
+
Source
{source_pass}/{source_total} pass
+
Owner
{owner}
+
+

{action}

+
+ """.format( + stage=html_text(item.get("stage", "")), + key=html_text(item.get("evidence_key", "")), + draft=html_text(item.get("draft_status", "")), + artifact_ready=html_text(item.get("artifact_ready_count", 0)), + artifact_total=html_text(item.get("artifact_total_count", 0)), + source_pass=html_text(item.get("source_pass_count", 0)), + source_total=html_text(item.get("source_check_count", 0)), + owner=html_text(item.get("owner", "")), + action=html_text(item.get("next_action", "")), + ) + for item in items + ) + + def render_html_item(item: dict[str, Any]) -> str: must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} runbook = must_collect.get("runbook", []) @@ -506,6 +557,7 @@ def render_html(report: dict[str, Any]) -> str: ] stat_html = "".join(f"
{html_text(label)}{html_text(value)}
" for label, value in stats) evidence_html = "".join(render_html_item(item) for item in report.get("evidence_items", [])) + matrix_html = render_html_matrix(report.get("evidence_matrix", [])) artifact_html = render_html_artifact_checklist(report.get("artifact_checklist", [])) source_html = render_html_source_checklist(report.get("source_checklist", [])) return f""" @@ -540,10 +592,12 @@ def render_html(report: dict[str, Any]) -> str: .section {{ padding:32px 0; border-bottom:1px solid var(--line); }} .panel {{ padding:20px; }} .two-col {{ display:grid; grid-template-columns:minmax(0, .45fr) minmax(0, 1fr); gap:18px; align-items:start; }} - .draft-grid, .evidence-grid, .artifact-grid, .source-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }} - .draft-card, .evidence-card, .artifact-card, .source-card {{ padding:18px; min-width:0; }} + .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .matrix-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }} + .draft-card, .evidence-card, .artifact-card, .source-card, .matrix-card {{ padding:18px; min-width:0; border:1px solid var(--line); border-radius:8px; background:#fff; }} .draft-card.written, .draft-card.exists {{ border-left:4px solid var(--pass); }} .draft-card.skipped {{ border-left:4px solid var(--warn); }} + .matrix-card.collect-source, .matrix-card.prepare-draft, .matrix-card.fix-artifacts, .matrix-card.fix-draft {{ border-left:4px solid var(--warn); }} + .matrix-card.validate-packet {{ border-left:4px solid var(--pass); }} .evidence-card.awaiting-submission, .evidence-card.fix-submission, .evidence-card.fix-template, .artifact-card.missing, .artifact-card.glob-no-match, .artifact-card.unsafe-path, .artifact-card.raw-content-disallowed {{ border-left:4px solid var(--warn); }} .artifact-card.ready {{ border-left:4px solid var(--pass); }} .source-card.blocked {{ border-left:4px solid var(--warn); }} @@ -561,11 +615,11 @@ def render_html(report: dict[str, Any]) -> str: .mini-grid li, .runbook-panel li, .notice li {{ overflow-wrap:anywhere; }} .notice {{ background:var(--soft); border-left:4px solid var(--ink); padding:16px; border-radius:8px; }} .errors {{ color:var(--warn); }} - @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} + @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .matrix-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} - +
Evidence Intake @@ -577,6 +631,7 @@ def render_html(report: dict[str, Any]) -> str:

Workflow

  1. Run the real provider, human review, native permission, or native client telemetry work first.
  2. Edit the matching JSON draft with aggregate artifact references and provenance metadata.
  3. Set template_only to false only after real evidence exists.
  4. Use prefilled SHA-256 values as convenience data, not evidence acceptance.
  5. Validate intake, refresh the ledger, then guard public claims.
+

Evidence Matrix

The matrix combines draft status, artifact readiness, source checks, and the next operator action. It is guidance only and never counts as accepted evidence.

{matrix_html}

Drafts

{render_html_files(report['files'])}

Artifact Checklist

Copy concrete paths and SHA-256 digests from here into artifact_refs after real evidence exists. Glob patterns are expanded for operator convenience only.

{artifact_html}

Source Evidence Snapshot

This section shows current aggregate source checks. It explains remaining blockers without changing the ledger.

{source_html}
@@ -617,6 +672,7 @@ def build_submission_kit( for item in items ] source_checklist = build_source_checklist(items) + evidence_matrix = build_evidence_matrix(items, files, artifact_checklist, source_checklist) manifest_path = output_dir / "submission_manifest.json" readme_path = output_dir / "README.md" output_html = output_html or (output_dir / "index.html") @@ -629,6 +685,7 @@ def build_submission_kit( artifact_missing_count = sum(1 for item in artifact_checklist if not item.get("artifact_ref_ready")) artifact_glob_count = sum(1 for item in artifact_checklist if item.get("concrete_reference_required")) source_summary = summarize_source_checklist(source_checklist) + matrix_summary = summarize_evidence_matrix(evidence_matrix) ok = not unknown_keys and skipped_count == 0 report = { "schema_version": "1.0", @@ -651,6 +708,7 @@ def build_submission_kit( "artifact_ref_prefill_count": prefilled_artifact_ref_count, "artifact_ref_unfilled_count": unfilled_artifact_ref_count, **source_summary, + **matrix_summary, "drafts_count_as_evidence": False, "ledger_counts_submission_as_completion": False, "decision": "submission-kit-ready" if ok else "fix-submission-kit", @@ -659,6 +717,7 @@ def build_submission_kit( "files": files, "artifact_checklist": artifact_checklist, "source_checklist": source_checklist, + "evidence_matrix": evidence_matrix, "evidence_items": items, "commands": { "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {shell_path(output_dir, skill_dir)}", diff --git a/scripts/world_class_submission_matrix.py b/scripts/world_class_submission_matrix.py new file mode 100644 index 0000000..42bb760 --- /dev/null +++ b/scripts/world_class_submission_matrix.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Build operator-facing readiness matrices for world-class submission kits.""" + +from collections import defaultdict +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Shared by submission kit rendering to summarize draft, artifact, and source-check readiness." + + +def _by_key(rows: list[dict[str, Any]], key_name: str = "evidence_key") -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + key = str(row.get(key_name, "")).strip() + if key: + grouped[key].append(row) + return dict(grouped) + + +def _draft_status(files_by_key: dict[str, list[dict[str, Any]]], key: str) -> tuple[str, str]: + rows = files_by_key.get(key, []) + if not rows: + return "missing", "" + row = rows[0] + return str(row.get("status", "missing")), str(row.get("output_path", "")) + + +def _first_blocked_action(source_rows: list[dict[str, Any]]) -> str: + for row in source_rows: + if row.get("status") != "pass": + action = str(row.get("next_action", "")).strip() + if action: + return action + return "" + + +def _stage( + *, + draft_status: str, + artifact_missing_count: int, + source_blocked_count: int, + invalid_draft: bool, +) -> str: + if draft_status == "skipped" or invalid_draft: + return "fix-draft" + if draft_status == "missing": + return "prepare-draft" + if source_blocked_count: + return "collect-source" + if artifact_missing_count: + return "fix-artifacts" + return "validate-packet" + + +def _next_action( + *, + stage: str, + draft_path: str, + first_blocked_action: str, + artifact_missing_count: int, +) -> str: + if stage == "fix-draft": + return "Fix skipped or invalid draft generation before evidence review." + if stage == "prepare-draft": + return "Generate the evidence draft, then keep template_only true until real evidence exists." + if stage == "collect-source": + return first_blocked_action or "Complete blocked source checks before ledger review." + if stage == "fix-artifacts": + return f"Resolve {artifact_missing_count} missing artifact reference(s) or replace globs with concrete paths." + if draft_path: + return f"Edit {draft_path}, set real provenance fields, validate intake, then refresh the ledger." + return "Validate intake, refresh the ledger, then run the public claim guard." + + +def build_evidence_matrix( + evidence_items: list[dict[str, Any]], + files: list[dict[str, Any]], + artifact_checklist: list[dict[str, Any]], + source_checklist: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Combine draft, artifact, and source-check readiness into one operator matrix.""" + files_by_key = _by_key(files) + artifacts_by_key = _by_key(artifact_checklist) + sources_by_key = _by_key(source_checklist) + matrix: list[dict[str, Any]] = [] + for item in evidence_items: + key = str(item.get("evidence_key", "")).strip() + if not key: + continue + artifact_rows = artifacts_by_key.get(key, []) + source_rows = sources_by_key.get(key, []) + draft_status, draft_path = _draft_status(files_by_key, key) + artifact_ready_count = sum(1 for row in artifact_rows if row.get("artifact_ref_ready")) + artifact_missing_count = len(artifact_rows) - artifact_ready_count + source_pass_count = sum(1 for row in source_rows if row.get("status") == "pass") + source_blocked_count = len(source_rows) - source_pass_count + invalid_draft = draft_status not in {"written", "exists"} + stage = _stage( + draft_status=draft_status, + artifact_missing_count=artifact_missing_count, + source_blocked_count=source_blocked_count, + invalid_draft=invalid_draft, + ) + first_action = _first_blocked_action(source_rows) + matrix.append( + { + "evidence_key": key, + "label": item.get("label", key), + "category": item.get("category", ""), + "owner": item.get("owner", ""), + "stage": stage, + "draft_status": draft_status, + "draft_path": draft_path, + "artifact_ready_count": artifact_ready_count, + "artifact_total_count": len(artifact_rows), + "artifact_missing_count": artifact_missing_count, + "source_pass_count": source_pass_count, + "source_check_count": len(source_rows), + "source_blocked_count": source_blocked_count, + "next_action": _next_action( + stage=stage, + draft_path=draft_path, + first_blocked_action=first_action, + artifact_missing_count=artifact_missing_count, + ), + "counts_as_completion": False, + } + ) + return matrix + + +def summarize_evidence_matrix(matrix: list[dict[str, Any]]) -> dict[str, int]: + return { + "evidence_matrix_count": len(matrix), + "evidence_matrix_collect_source_count": sum(1 for item in matrix if item.get("stage") == "collect-source"), + "evidence_matrix_prepare_draft_count": sum(1 for item in matrix if item.get("stage") == "prepare-draft"), + "evidence_matrix_fix_artifacts_count": sum(1 for item in matrix if item.get("stage") == "fix-artifacts"), + "evidence_matrix_validate_packet_count": sum(1 for item in matrix if item.get("stage") == "validate-packet"), + "evidence_matrix_counts_as_completion": 0, + } diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index 18a28f7..fd23249 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -129,6 +129,7 @@ "scripts/verify_package.py", "scripts/world_class_evidence_contract.py", "scripts/world_class_source_checks.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index fdc88ce..f57d826 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -620,10 +620,25 @@ def main() -> None: assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"] assert kit_payload["summary"]["source_pass_count"] == 1, kit_payload["summary"] assert kit_payload["summary"]["source_blocked_count"] == 2, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_collect_source_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_counts_as_completion"] == 0, kit_payload["summary"] assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"] assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"] assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"] assert kit_payload["files"][0]["output_path"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/provider-holdout.json"), kit_payload["files"] + assert len(kit_payload["evidence_matrix"]) == 1, kit_payload["evidence_matrix"] + matrix_row = kit_payload["evidence_matrix"][0] + assert matrix_row["evidence_key"] == "provider-holdout", matrix_row + assert matrix_row["stage"] == "collect-source", matrix_row + assert matrix_row["draft_status"] == "written", matrix_row + assert matrix_row["artifact_ready_count"] >= 1, matrix_row + assert matrix_row["artifact_total_count"] >= matrix_row["artifact_ready_count"], matrix_row + assert matrix_row["source_pass_count"] == 1, matrix_row + assert matrix_row["source_check_count"] == 3, matrix_row + assert matrix_row["source_blocked_count"] == 2, matrix_row + assert matrix_row["counts_as_completion"] is False, matrix_row + assert "real credentials" in matrix_row["next_action"], matrix_row artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]} assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows @@ -651,6 +666,9 @@ def main() -> None: assert "output-exec --provider-runner openai" in kit_readme, kit_readme assert "validate intake" in kit_readme, kit_readme assert "Artifact Checklist" in kit_readme, kit_readme + assert "Evidence Matrix" in kit_readme, kit_readme + assert "`collect-source`" in kit_readme, kit_readme + assert "Matrix rows are guidance only" in kit_readme, kit_readme assert "Source Evidence Snapshot" in kit_readme, kit_readme assert "reports/output_execution_runs.json" in kit_readme, kit_readme assert "Provider model run" in kit_readme, kit_readme @@ -659,6 +677,13 @@ def main() -> None: assert "Drafts are not accepted evidence" in kit_html, kit_html assert "provider-holdout" in kit_html, kit_html assert "Artifact Checklist" in kit_html, kit_html + assert "Evidence Matrix" in kit_html, kit_html + assert "matrix-card collect-source" in kit_html, kit_html + assert ( + f"
Artifacts
{matrix_row['artifact_ready_count']}/{matrix_row['artifact_total_count']} ready
" + in kit_html + ), kit_html + assert "
Source
1/3 pass
" in kit_html, kit_html assert "Source Evidence Snapshot" in kit_html, kit_html assert "
Field
model_executed_count
" in kit_html, kit_html assert "
Current
0
" in kit_html, kit_html