diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index c9c3dd0..33012b9 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,10 +4,10 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 211, - "script_file_count": 137, + "python_file_count": 212, + "script_file_count": 138, "test_file_count": 74, - "internal_module_count": 53, + "internal_module_count": 54, "cli_script_count": 86, "command_handler_count": 68, "entrypoint_command_handler_count": 18, @@ -37,15 +37,15 @@ "recommendation": "Split viewer data assembly from HTML section rendering." }, { - "path": "scripts/skill_report_model.py", - "lines": 665, - "kind": "internal-module", + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 677, + "kind": "test", "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { - "path": "scripts/world_class_evidence_contract.py", - "lines": 654, + "path": "scripts/skill_report_model.py", + "lines": 665, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." @@ -93,11 +93,11 @@ "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 611, - "kind": "test", + "path": "scripts/world_class_evidence_contract.py", + "lines": 618, + "kind": "internal-module", "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { "path": "scripts/render_benchmark_reproducibility.py", diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 7b75ccc..4e0fc49 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,10 +5,10 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `211` -- scripts: `137` +- python files: `212` +- scripts: `138` - tests: `74` -- internal modules: `53` +- internal modules: `54` - CLI scripts: `86` - Yao CLI command handlers: `68` - entrypoint command handlers: `18` @@ -35,15 +35,15 @@ No near-threshold files found. | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `696` | `test` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | +| `tests/verify_world_class_evidence_intake.py` | `677` | `test` | `pass` | | `scripts/skill_report_model.py` | `665` | `internal-module` | `pass` | -| `scripts/world_class_evidence_contract.py` | `654` | `internal-module` | `pass` | | `scripts/render_skill_os2_coverage.py` | `649` | `cli-script` | `pass` | | `scripts/render_review_studio.py` | `647` | `cli-script` | `pass` | | `scripts/render_reference_synthesis.py` | `644` | `cli-script` | `pass` | | `scripts/review_studio_gates.py` | `643` | `internal-module` | `pass` | | `scripts/cross_packager.py` | `638` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `637` | `cli-script` | `pass` | -| `tests/verify_world_class_evidence_intake.py` | `611` | `test` | `pass` | +| `scripts/world_class_evidence_contract.py` | `618` | `internal-module` | `pass` | | `scripts/render_benchmark_reproducibility.py` | `595` | `cli-script` | `pass` | ## Release Rule diff --git a/reports/context_budget.json b/reports/context_budget.json index bf3c866..1ac6299 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1064742, + "other_text_tokens": 1067291, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1065539, - "deferred_resource_tokens": 493019, + "estimated_total_text_tokens": 1068088, + "deferred_resource_tokens": 494370, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 432921, - "file_count": 137 + "estimated_tokens": 434272, + "file_count": 138 }, { "path": "references", @@ -36,8 +36,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 432921, - "file_count": 137 + "estimated_tokens": 434272, + "file_count": 138 } ], "deferred_resource_governance": { @@ -59,14 +59,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 432921, - "file_count": 137, + "estimated_tokens": 434272, + "file_count": 138, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 669, + "relevant_file_count": 671, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index afd7ad1..7d56a3a 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 493019 | `governed` | scripts:432921 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 494370 | `governed` | scripts:434272 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 6d2fa88..ce753f7 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-06-16", + "generated_at": "2026-03-31", "targets": [ { "label": "root", @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 493019, + "deferred_resource_tokens": 494370, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 432921, - "file_count": 137 + "estimated_tokens": 434272, + "file_count": 138 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 432921, - "file_count": 137, + "estimated_tokens": 434272, + "file_count": 138, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index f5e37eb..6515f38 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 214, + "file_count": 215, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -760,6 +760,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/world_class_preflight_layout.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 9d1c8aa..f11e1d7 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-16` - decision: `pass` - target python: `3.11` -- files scanned: `214` +- files scanned: `215` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index c911430..aa724e2 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 228, - "script_count": 137, - "internal_module_count": 51, + "scanned_files": 229, + "script_count": 138, + "internal_module_count": 52, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 228, - "package_sha256": "662e1fd1592dfdea9053d3e1e6458c81d71c45a2900dc7814a557fcd8e31f4e0" + "package_hash_file_count": 229, + "package_sha256": "2bc5e8214814c344534e5155301cd3f86e4ceee351313fb3a538d6fd9d400b75" }, "failures": [], "warnings": [], @@ -1726,6 +1726,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/world_class_preflight_layout.py", "interface": "internal-module", @@ -1999,7 +2013,7 @@ "checked_count": 86, "passed_count": 86, "failed_count": 0, - "skipped_count": 51, + "skipped_count": 52, "failed_scripts": [], "results": [ { @@ -3004,6 +3018,10 @@ "path": "scripts/world_class_human_evidence.py", "reason": "internal module" }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "reason": "internal module" + }, { "path": "scripts/world_class_preflight_layout.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 1b6c026..557ac00 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `228` -- Scripts: `137` -- Internal script modules: `51` +- Scanned files: `229` +- Scripts: `138` +- Internal script modules: `52` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `228` -- Package SHA256: `662e1fd1592dfdea9053d3e1e6458c81d71c45a2900dc7814a557fcd8e31f4e0` +- Package hash files: `229` +- Package SHA256: `2bc5e8214814c344534e5155301cd3f86e4ceee351313fb3a538d6fd9d400b75` ## Failures @@ -182,6 +182,7 @@ | scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. | | scripts/world_class_human_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows. | +| scripts/world_class_native_permission_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows. | | scripts/world_class_preflight_layout.py | internal-module | True | False | False | False | False | False | False | Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly. | | scripts/world_class_provider_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows. | | scripts/world_class_source_checks.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to keep source-evidence readiness checks consistent. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 8288708..2b2f91c 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -22,6 +22,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps. - World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields. - World-Class Provider Evidence Guard v0 so provider holdout submissions must reconcile `summary` counts with `runs` rows and include at least one passing model run whose provider, model, timing, non-estimated usage, and output hash match the submitted provenance. +- World-Class Native Permission Evidence Guard v0 so native-permission submissions must reconcile runtime probe summary counts with target rows, require at least one native-enforced target row, and keep installer permission checks failure-free. - World-Class Submission Matrix v0 so the submission kit exposes a single operator-facing matrix for draft status, artifact readiness, source-check blockers, and the next action without counting matrix rows as completion evidence. - World-Class Submission Artifact Roles v0 so submission kits distinguish `submission-ref` rows that belong in `artifact_refs` from supporting audit assets, reducing operator ambiguity without accepting evidence. - World-Class Submission Kit Rendering v0 so Markdown and HTML kit presentation lives in a dedicated internal renderer, keeping the CLI focused on evidence assembly and file emission. @@ -67,7 +68,7 @@ This is still not the final world-class state. Target-native behavior contracts | Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | | World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | | World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed | -| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_provider_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, provider run-row validation, and nested raw-field rejection | v0 landed | +| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, provider run-row validation, native permission target-row validation, and nested raw-field rejection | v0 landed | | World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | @@ -128,7 +129,7 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `51` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `137` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `52` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `138` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index b402372..5dde601 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 137; help failures 0", + "current": "secrets 0; scripts 138; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index bb985cf..60c19dd 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,7 +23,7 @@ Generated at: `2026-06-16` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 137; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 138; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 5968213..8ad273a 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "137 scripts; secrets 0; help failures 0", + "current": "138 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index a36caa8..0b97511 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -26,7 +26,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 137 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 138 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | | Registry Distribution | `pass` | archive entries 658; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | diff --git a/scripts/world_class_evidence_contract.py b/scripts/world_class_evidence_contract.py index 87c3934..4fa978a 100644 --- a/scripts/world_class_evidence_contract.py +++ b/scripts/world_class_evidence_contract.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any from world_class_human_evidence import validate_human_adjudication_report +from world_class_native_permission_evidence import validate_native_permission_report from world_class_provider_evidence import validate_provider_execution_report @@ -411,44 +412,7 @@ def validate_native_permission_artifacts(payload: dict[str, Any], errors: list[s probes = load_json(paths.get("reports/runtime_permission_probes.json", root / "__missing__")) install = load_json(paths.get("reports/install_simulation.json", root / "__missing__")) if probes: - probe_summary = summary(probes) - add_error(errors, probes.get("ok") is True, "native-permission-enforcement runtime probe report ok must be true") - add_error( - errors, - bool(real_int(probe_summary.get("native_enforcement_count")) and probe_summary["native_enforcement_count"] > 0), - "native-permission-enforcement runtime probe summary.native_enforcement_count must be >0", - ) - add_error( - errors, - probe_summary.get("failure_count") == 0, - "native-permission-enforcement runtime probe summary.failure_count must be 0", - ) - add_error( - errors, - probe_summary.get("installer_enforcement_ready") is True, - "native-permission-enforcement runtime probe summary.installer_enforcement_ready must be true", - ) - if install: - install_summary = summary(install) - add_error(errors, install.get("ok") is True, "native-permission-enforcement install simulation report ok must be true") - add_error( - errors, - bool( - real_int(install_summary.get("installer_permission_enforced_count")) - and install_summary["installer_permission_enforced_count"] > 0 - ), - "native-permission-enforcement install simulation summary.installer_permission_enforced_count must be >0", - ) - add_error( - errors, - install_summary.get("installer_permission_failure_count") == 0, - "native-permission-enforcement install simulation summary.installer_permission_failure_count must be 0", - ) - add_error( - errors, - install_summary.get("failure_count") == 0, - "native-permission-enforcement install simulation summary.failure_count must be 0", - ) + validate_native_permission_report(probes, install, errors) def validate_native_client_telemetry_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: diff --git a/scripts/world_class_native_permission_evidence.py b/scripts/world_class_native_permission_evidence.py new file mode 100644 index 0000000..c64de04 --- /dev/null +++ b/scripts/world_class_native_permission_evidence.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows." + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def object_list(value: Any) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def sorted_strings(value: Any) -> list[str]: + return sorted(str(item) for item in value) if isinstance(value, list) else [] + + +def validate_target_rows(probes: dict[str, Any], summary: dict[str, Any], errors: list[str]) -> None: + targets = object_list(probes.get("targets", [])) + target_count = real_int(summary.get("target_count")) + native_count = real_int(summary.get("native_enforcement_count")) + fail_count = real_int(summary.get("fail_count")) + expected_capabilities = sorted_strings(probes.get("expected_capabilities")) + + add_error(errors, bool(targets), "native-permission-enforcement runtime probe targets must not be empty") + if target_count is not None: + add_error(errors, len(targets) == target_count, "native-permission-enforcement runtime probe targets length must equal summary.target_count") + add_error(errors, fail_count == 0, "native-permission-enforcement runtime probe summary.fail_count must be 0") + + observed_native = 0 + for index, target in enumerate(targets, start=1): + target_name = str(target.get("target", "")).strip() or str(index) + add_error(errors, target.get("status") == "pass", f"native-permission-enforcement target {target_name} status must be pass") + add_error(errors, not target.get("failures"), f"native-permission-enforcement target {target_name} failures must be empty") + checks = object_list(target.get("checks", [])) + add_error(errors, bool(checks), f"native-permission-enforcement target {target_name} checks must not be empty") + add_error( + errors, + all(item.get("passed") is True for item in checks), + f"native-permission-enforcement target {target_name} checks must all pass", + ) + native = target.get("native_enforcement") is True + if native: + observed_native += 1 + add_error( + errors, + target.get("assurance") == "native-enforced", + f"native-permission-enforcement target {target_name} assurance must be native-enforced", + ) + else: + add_error( + errors, + target.get("metadata_fallback_explicit") is True, + f"native-permission-enforcement fallback target {target_name} must keep metadata_fallback_explicit true", + ) + add_error( + errors, + bool(target.get("residual_risks")), + f"native-permission-enforcement fallback target {target_name} must retain residual risks", + ) + if expected_capabilities: + add_error( + errors, + sorted_strings(target.get("declared_capabilities")) == expected_capabilities, + f"native-permission-enforcement target {target_name} declared_capabilities must match expected_capabilities", + ) + installer = target.get("installer_enforcement", {}) if isinstance(target.get("installer_enforcement"), dict) else {} + add_error( + errors, + installer.get("enforced") is True, + f"native-permission-enforcement target {target_name} installer_enforcement.enforced must be true", + ) + add_error( + errors, + not installer.get("missing_capabilities"), + f"native-permission-enforcement target {target_name} installer_enforcement missing_capabilities must be empty", + ) + add_error( + errors, + not installer.get("failure_details"), + f"native-permission-enforcement target {target_name} installer_enforcement failure_details must be empty", + ) + + add_error(errors, bool(native_count and observed_native == native_count), "native-permission-enforcement native target rows must match summary.native_enforcement_count") + add_error(errors, observed_native > 0, "native-permission-enforcement must include at least one native-enforced target row") + + +def validate_install_simulation_report(install: dict[str, Any], errors: list[str]) -> None: + summary = install.get("summary", {}) if isinstance(install.get("summary", {}), dict) else {} + add_error(errors, install.get("ok") is True, "native-permission-enforcement install simulation report ok must be true") + add_error( + errors, + bool(real_int(summary.get("installer_permission_enforced_count")) and summary["installer_permission_enforced_count"] > 0), + "native-permission-enforcement install simulation summary.installer_permission_enforced_count must be >0", + ) + add_error( + errors, + summary.get("installer_permission_failure_count") == 0, + "native-permission-enforcement install simulation summary.installer_permission_failure_count must be 0", + ) + add_error(errors, summary.get("failure_count") == 0, "native-permission-enforcement install simulation summary.failure_count must be 0") + permission_checks = [ + item + for item in object_list(install.get("checks", [])) + if str(item.get("id", "")).startswith("permission-") and str(item.get("id", "")).endswith(("-approved", "-target-enforcement")) + ] + if permission_checks: + add_error( + errors, + all(item.get("status") == "pass" for item in permission_checks), + "native-permission-enforcement install simulation permission checks must all pass", + ) + + +def validate_native_permission_report(probes: dict[str, Any], install: dict[str, Any], errors: list[str]) -> None: + summary = probes.get("summary", {}) if isinstance(probes.get("summary", {}), dict) else {} + add_error(errors, probes.get("ok") is True, "native-permission-enforcement runtime probe report ok must be true") + add_error( + errors, + bool(real_int(summary.get("native_enforcement_count")) and summary["native_enforcement_count"] > 0), + "native-permission-enforcement runtime probe summary.native_enforcement_count must be >0", + ) + add_error( + errors, + summary.get("failure_count") == 0, + "native-permission-enforcement runtime probe summary.failure_count must be 0", + ) + add_error( + errors, + summary.get("installer_enforcement_ready") is True, + "native-permission-enforcement runtime probe summary.installer_enforcement_ready must be true", + ) + validate_target_rows(probes, summary, errors) + if install: + validate_install_simulation_report(install, errors) diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index e378f9b..7c9794d 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -201,16 +201,47 @@ def provider_artifact_submission(skill_root: Path) -> dict: def write_native_permission_artifacts(skill_root: Path, *, complete: bool) -> None: + capabilities = ["file_write", "network", "subprocess"] + target = { + "target": "vscode", + "status": "pass", + "adapter": "dist/targets/vscode/adapter.json", + "permission_model": "native-client", + "native_enforcement": complete, + "metadata_fallback_explicit": not complete, + "installer_enforcement": { + "target": "vscode", + "source_status": "present", + "enforced": True, + "enforced_capabilities": capabilities, + "missing_capabilities": [], + "failure_details": [], + }, + "assurance": "native-enforced" if complete else "metadata-fallback-explicit", + "declared_capabilities": capabilities, + "checks": [ + {"key": "adapter-present", "passed": True, "detail": "vscode adapter.json is readable"}, + {"key": "native-enforcement-boolean", "passed": True, "detail": "native flag is boolean"}, + ], + "failures": [], + "residual_risks": [] if complete else ["Client-native permission enforcement is not provided by this target."], + } write_json( skill_root / "reports" / "runtime_permission_probes.json", { "schema_version": "1.0", "ok": True, + "expected_capabilities": capabilities, "summary": { + "target_count": 1, + "pass_count": 1, + "fail_count": 0, "native_enforcement_count": 1 if complete else 0, + "metadata_fallback_count": 0 if complete else 1, "failure_count": 0, "installer_enforcement_ready": True, }, + "targets": [target], }, ) write_json( @@ -221,8 +252,26 @@ def write_native_permission_artifacts(skill_root: Path, *, complete: bool) -> No "summary": { "installer_permission_enforced_count": 12, "installer_permission_failure_count": 0, + "permission_target_count": 1, + "permission_capability_count": 3, "failure_count": 0, }, + "checks": [ + {"id": "permission-vscode-file_write-approved", "status": "pass", "detail": "file write approved"}, + { + "id": "permission-vscode-file_write-target-enforcement", + "status": "pass", + "detail": "file write enforced", + }, + {"id": "permission-vscode-network-approved", "status": "pass", "detail": "network approved"}, + {"id": "permission-vscode-network-target-enforcement", "status": "pass", "detail": "network enforced"}, + {"id": "permission-vscode-subprocess-approved", "status": "pass", "detail": "subprocess approved"}, + { + "id": "permission-vscode-subprocess-target-enforcement", + "status": "pass", + "detail": "subprocess enforced", + }, + ], }, ) @@ -415,6 +464,23 @@ def assert_external_contract_artifact_validation() -> None: template_expected=False, ) assert permission_valid["status"] == "pass", permission_valid + forged_probe = json.loads((skill_root / "reports" / "runtime_permission_probes.json").read_text(encoding="utf-8")) + forged_probe["targets"][0]["native_enforcement"] = False + forged_probe["targets"][0]["assurance"] = "metadata-fallback-explicit" + forged_probe["targets"][0]["metadata_fallback_explicit"] = True + forged_probe["targets"][0]["residual_risks"] = ["Fallback still requires operator enforcement."] + write_json(skill_root / "reports" / "runtime_permission_probes.json", forged_probe) + permission_forged = validate_payload( + native_permission_submission(skill_root), + permission_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-permission-enforcement.json", + root=skill_root, + template_expected=False, + ) + assert permission_forged["status"] == "fail", permission_forged + assert any("native target rows must match summary.native_enforcement_count" in error for error in permission_forged["errors"]), ( + permission_forged["errors"] + ) write_native_permission_artifacts(skill_root, complete=False) permission_invalid = validate_payload( native_permission_submission(skill_root),