From 03aa6287123b4f47aad2a7abd59bb950e4d577e6 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 23:37:57 +0800 Subject: [PATCH] Validate native telemetry evidence rows --- reports/architecture_maintainability.json | 30 +-- reports/architecture_maintainability.md | 12 +- reports/context_budget.json | 20 +- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 10 +- reports/python_compatibility.json | 14 +- reports/python_compatibility.md | 2 +- reports/security_trust_report.json | 30 ++- reports/security_trust_report.md | 11 +- reports/skill-os-2-review.md | 5 +- reports/skill_os2_audit.json | 2 +- reports/skill_os2_audit.md | 2 +- reports/skill_os2_coverage.json | 2 +- reports/skill_os2_coverage.md | 2 +- scripts/world_class_evidence_contract.py | 62 +----- .../world_class_native_telemetry_evidence.py | 130 ++++++++++++ tests/verify_world_class_evidence_intake.py | 71 +------ .../world_class_native_telemetry_fixtures.py | 185 ++++++++++++++++++ 18 files changed, 407 insertions(+), 185 deletions(-) create mode 100644 scripts/world_class_native_telemetry_evidence.py create mode 100644 tests/world_class_native_telemetry_fixtures.py diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index 33012b99..bbe1faac 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,10 +4,10 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 212, - "script_file_count": 138, - "test_file_count": 74, - "internal_module_count": 54, + "python_file_count": 214, + "script_file_count": 139, + "test_file_count": 75, + "internal_module_count": 55, "cli_script_count": 86, "command_handler_count": 68, "entrypoint_command_handler_count": 18, @@ -36,13 +36,6 @@ "severity": "pass", "recommendation": "Split viewer data assembly from HTML section rendering." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 677, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "scripts/skill_report_model.py", "lines": 665, @@ -93,11 +86,11 @@ "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, { - "path": "scripts/world_class_evidence_contract.py", - "lines": 618, - "kind": "internal-module", + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 610, + "kind": "test", "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, { "path": "scripts/render_benchmark_reproducibility.py", @@ -105,6 +98,13 @@ "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/optimize_description.py", + "lines": 585, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." } ], "watchlist": [], diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 4e0fc49c..e4f40ed1 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,10 +5,10 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `212` -- scripts: `138` -- tests: `74` -- internal modules: `54` +- python files: `214` +- scripts: `139` +- tests: `75` +- internal modules: `55` - CLI scripts: `86` - Yao CLI command handlers: `68` - entrypoint command handlers: `18` @@ -35,7 +35,6 @@ No near-threshold files found. | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `696` | `test` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | -| `tests/verify_world_class_evidence_intake.py` | `677` | `test` | `pass` | | `scripts/skill_report_model.py` | `665` | `internal-module` | `pass` | | `scripts/render_skill_os2_coverage.py` | `649` | `cli-script` | `pass` | | `scripts/render_review_studio.py` | `647` | `cli-script` | `pass` | @@ -43,8 +42,9 @@ No near-threshold files found. | `scripts/review_studio_gates.py` | `643` | `internal-module` | `pass` | | `scripts/cross_packager.py` | `638` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `637` | `cli-script` | `pass` | -| `scripts/world_class_evidence_contract.py` | `618` | `internal-module` | `pass` | +| `tests/verify_world_class_evidence_intake.py` | `610` | `test` | `pass` | | `scripts/render_benchmark_reproducibility.py` | `595` | `cli-script` | `pass` | +| `scripts/optimize_description.py` | `585` | `cli-script` | `pass` | ## Release Rule diff --git a/reports/context_budget.json b/reports/context_budget.json index 1ac62999..461f4746 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1067291, + "other_text_tokens": 1069589, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1068088, - "deferred_resource_tokens": 494370, + "estimated_total_text_tokens": 1070386, + "deferred_resource_tokens": 495216, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 434272, - "file_count": 138 + "estimated_tokens": 435118, + "file_count": 139 }, { "path": "references", @@ -36,8 +36,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 434272, - "file_count": 138 + "estimated_tokens": 435118, + "file_count": 139 } ], "deferred_resource_governance": { @@ -59,14 +59,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 434272, - "file_count": 138, + "estimated_tokens": 435118, + "file_count": 139, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 671, + "relevant_file_count": 675, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index 7d56a3ac..e15688a3 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 494370 | `governed` | scripts:434272 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 495216 | `governed` | scripts:435118 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index e59374a7..f23d0898 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 494370, + "deferred_resource_tokens": 495216, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 434272, - "file_count": 138 + "estimated_tokens": 435118, + "file_count": 139 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 434272, - "file_count": 138, + "estimated_tokens": 435118, + "file_count": 139, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 6515f380..282d30f8 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 215, + "file_count": 217, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -766,6 +766,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/world_class_preflight_layout.py", "ok": true, @@ -1300,6 +1306,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/world_class_native_telemetry_fixtures.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/yao_cli_helpers.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index f11e1d74..b03b8135 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-16` - decision: `pass` - target python: `3.11` -- files scanned: `215` +- files scanned: `217` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index aa724e24..40d1d9d5 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 229, - "script_count": 138, - "internal_module_count": 52, + "scanned_files": 230, + "script_count": 139, + "internal_module_count": 53, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 229, - "package_sha256": "2bc5e8214814c344534e5155301cd3f86e4ceee351313fb3a538d6fd9d400b75" + "package_hash_file_count": 230, + "package_sha256": "79e555f12cb95ad51753c69fdb7fce367f3a394c3e55c075ab4754feb6d3200b" }, "failures": [], "warnings": [], @@ -1740,6 +1740,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/world_class_preflight_layout.py", "interface": "internal-module", @@ -2013,7 +2027,7 @@ "checked_count": 86, "passed_count": 86, "failed_count": 0, - "skipped_count": 52, + "skipped_count": 53, "failed_scripts": [], "results": [ { @@ -3022,6 +3036,10 @@ "path": "scripts/world_class_native_permission_evidence.py", "reason": "internal module" }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "reason": "internal module" + }, { "path": "scripts/world_class_preflight_layout.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 557ac005..6d349620 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `229` -- Scripts: `138` -- Internal script modules: `52` +- Scanned files: `230` +- Scripts: `139` +- Internal script modules: `53` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `229` -- Package SHA256: `2bc5e8214814c344534e5155301cd3f86e4ceee351313fb3a538d6fd9d400b75` +- Package hash files: `230` +- Package SHA256: `79e555f12cb95ad51753c69fdb7fce367f3a394c3e55c075ab4754feb6d3200b` ## Failures @@ -183,6 +183,7 @@ | scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. | | scripts/world_class_human_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows. | | scripts/world_class_native_permission_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows. | +| scripts/world_class_native_telemetry_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows. | | scripts/world_class_preflight_layout.py | internal-module | True | False | False | False | False | False | False | Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly. | | scripts/world_class_provider_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows. | | scripts/world_class_source_checks.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to keep source-evidence readiness checks consistent. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 2b2f91cb..836c5556 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -23,6 +23,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields. - World-Class Provider Evidence Guard v0 so provider holdout submissions must reconcile `summary` counts with `runs` rows and include at least one passing model run whose provider, model, timing, non-estimated usage, and output hash match the submitted provenance. - World-Class Native Permission Evidence Guard v0 so native-permission submissions must reconcile runtime probe summary counts with target rows, require at least one native-enforced target row, and keep installer permission checks failure-free. +- World-Class Native Telemetry Evidence Guard v0 so native-client-telemetry submissions must reconcile adoption summary counts with external metadata event rows and keep hook recipes metadata-only without claiming native auto-capture. - World-Class Submission Matrix v0 so the submission kit exposes a single operator-facing matrix for draft status, artifact readiness, source-check blockers, and the next action without counting matrix rows as completion evidence. - World-Class Submission Artifact Roles v0 so submission kits distinguish `submission-ref` rows that belong in `artifact_refs` from supporting audit assets, reducing operator ambiguity without accepting evidence. - World-Class Submission Kit Rendering v0 so Markdown and HTML kit presentation lives in a dedicated internal renderer, keeping the CLI focused on evidence assembly and file emission. @@ -68,7 +69,7 @@ This is still not the final world-class state. Target-native behavior contracts | Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | | World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | | World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed | -| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, provider run-row validation, native permission target-row validation, and nested raw-field rejection | v0 landed | +| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/world_class_native_telemetry_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, provider run-row validation, native permission target-row validation, native telemetry event-row validation, and nested raw-field rejection | v0 landed | | World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | @@ -129,7 +130,7 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `52` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `138` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `53` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `139` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 5dde601a..8d6e7169 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 138; help failures 0", + "current": "secrets 0; scripts 139; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 60c19ddc..dfa19ee1 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,7 +23,7 @@ Generated at: `2026-06-16` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 138; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 139; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 8ad273a6..47597310 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "138 scripts; secrets 0; help failures 0", + "current": "139 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 0b97511e..cd32cba7 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -26,7 +26,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 138 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 139 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | | Registry Distribution | `pass` | archive entries 658; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | diff --git a/scripts/world_class_evidence_contract.py b/scripts/world_class_evidence_contract.py index 4fa978af..bfb3bba8 100644 --- a/scripts/world_class_evidence_contract.py +++ b/scripts/world_class_evidence_contract.py @@ -8,6 +8,7 @@ from typing import Any from world_class_human_evidence import validate_human_adjudication_report from world_class_native_permission_evidence import validate_native_permission_report +from world_class_native_telemetry_evidence import validate_native_telemetry_report from world_class_provider_evidence import validate_provider_execution_report @@ -369,25 +370,6 @@ def artifact_ref_path_map(payload: dict[str, Any], root: Path) -> dict[str, Path return paths -def real_int(value: Any) -> int | None: - return value if isinstance(value, int) and not isinstance(value, bool) else None - - -def summary(payload: dict[str, Any]) -> dict[str, Any]: - value = payload.get("summary", {}) - return value if isinstance(value, dict) else {} - - -def source_types(payload: dict[str, Any]) -> dict[str, Any]: - value = summary(payload).get("source_types", {}) - return value if isinstance(value, dict) else {} - - -def privacy_contract(payload: dict[str, Any]) -> dict[str, Any]: - value = payload.get("privacy_contract", {}) - return value if isinstance(value, dict) else {} - - def validate_provider_holdout_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: paths = artifact_ref_path_map(payload, root) execution = load_json(paths.get("reports/output_execution_runs.json", root / "__missing__")) @@ -419,47 +401,7 @@ def validate_native_client_telemetry_artifacts(payload: dict[str, Any], errors: paths = artifact_ref_path_map(payload, root) adoption = load_json(paths.get("reports/adoption_drift_report.json", root / "__missing__")) recipes = load_json(paths.get("reports/telemetry_hook_recipes.json", root / "__missing__")) - if adoption: - adoption_summary = summary(adoption) - adoption_privacy = privacy_contract(adoption) - add_error(errors, adoption.get("ok") is True, "native-client-telemetry adoption drift report ok must be true") - add_error( - errors, - bool(real_int(source_types(adoption).get("external")) and source_types(adoption)["external"] > 0), - "native-client-telemetry adoption drift summary.source_types.external must be >0", - ) - add_error( - errors, - bool(real_int(adoption_summary.get("adoption_sample_count")) and adoption_summary["adoption_sample_count"] > 0), - "native-client-telemetry adoption drift summary.adoption_sample_count must be >0", - ) - add_error( - errors, - adoption_privacy.get("raw_content_allowed") is False, - "native-client-telemetry adoption drift privacy_contract.raw_content_allowed must be false", - ) - add_error( - errors, - adoption_privacy.get("raw_event_log_packaged") is False, - "native-client-telemetry adoption drift privacy_contract.raw_event_log_packaged must be false", - ) - if recipes: - recipes_summary = summary(recipes) - recipes_privacy = privacy_contract(recipes) - add_error(errors, recipes.get("ok") is True, "native-client-telemetry hook recipes report ok must be true") - add_error( - errors, - bool( - real_int(recipes_summary.get("metadata_only_recipe_count")) - and recipes_summary["metadata_only_recipe_count"] > 0 - ), - "native-client-telemetry hook recipes summary.metadata_only_recipe_count must be >0", - ) - add_error( - errors, - recipes_privacy.get("raw_content_allowed") is False, - "native-client-telemetry hook recipes privacy_contract.raw_content_allowed must be false", - ) + validate_native_telemetry_report(adoption, recipes, errors) def validate_real_artifact_payloads(payload: dict[str, Any], errors: list[str], root: Path, template_expected: bool) -> None: diff --git a/scripts/world_class_native_telemetry_evidence.py b/scripts/world_class_native_telemetry_evidence.py new file mode 100644 index 00000000..76d6874f --- /dev/null +++ b/scripts/world_class_native_telemetry_evidence.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows." + +ADOPTION_EVENTS = {"skill_activation", "skill_output"} + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def object_list(value: Any) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def source_types(payload: dict[str, Any]) -> dict[str, Any]: + summary = payload.get("summary", {}) if isinstance(payload.get("summary"), dict) else {} + value = summary.get("source_types", {}) + return value if isinstance(value, dict) else {} + + +def validate_adoption_report(adoption: dict[str, Any], errors: list[str]) -> None: + summary = adoption.get("summary", {}) if isinstance(adoption.get("summary"), dict) else {} + privacy = adoption.get("privacy_contract", {}) if isinstance(adoption.get("privacy_contract"), dict) else {} + events = object_list(adoption.get("recent_events", [])) + adoption_rows = object_list(adoption.get("adoption_by_skill", [])) + external_summary_count = real_int(source_types(adoption).get("external")) + adoption_sample_count = real_int(summary.get("adoption_sample_count")) + + add_error(errors, adoption.get("ok") is True, "native-client-telemetry adoption drift report ok must be true") + add_error( + errors, + bool(external_summary_count and external_summary_count > 0), + "native-client-telemetry adoption drift summary.source_types.external must be >0", + ) + add_error( + errors, + bool(adoption_sample_count and adoption_sample_count > 0), + "native-client-telemetry adoption drift summary.adoption_sample_count must be >0", + ) + add_error( + errors, + privacy.get("raw_content_allowed") is False, + "native-client-telemetry adoption drift privacy_contract.raw_content_allowed must be false", + ) + add_error( + errors, + privacy.get("raw_event_log_packaged") is False, + "native-client-telemetry adoption drift privacy_contract.raw_event_log_packaged must be false", + ) + add_error(errors, bool(events), "native-client-telemetry adoption drift recent_events must not be empty") + external_events = [item for item in events if item.get("source") == "external"] + external_adoption_events = [item for item in external_events if item.get("event") in ADOPTION_EVENTS] + add_error( + errors, + bool(external_summary_count and len(external_events) >= external_summary_count), + "native-client-telemetry external event rows must cover summary.source_types.external", + ) + add_error( + errors, + bool(external_adoption_events), + "native-client-telemetry must include at least one external adoption event row", + ) + add_error( + errors, + bool( + adoption_sample_count + and len([item for item in events if item.get("event") in ADOPTION_EVENTS]) >= adoption_sample_count + ), + "native-client-telemetry adoption event rows must cover summary.adoption_sample_count", + ) + add_error( + errors, + any((count := real_int(row.get("adoption_events"))) is not None and count > 0 for row in adoption_rows), + "native-client-telemetry adoption_by_skill must include at least one adoption event", + ) + + +def validate_recipe_report(recipes: dict[str, Any], errors: list[str]) -> None: + summary = recipes.get("summary", {}) if isinstance(recipes.get("summary"), dict) else {} + privacy = recipes.get("privacy_contract", {}) if isinstance(recipes.get("privacy_contract"), dict) else {} + recipe_rows = object_list(recipes.get("recipes", [])) + recipe_count = real_int(summary.get("recipe_count")) + metadata_count = real_int(summary.get("metadata_only_recipe_count")) + + add_error(errors, recipes.get("ok") is True, "native-client-telemetry hook recipes report ok must be true") + add_error( + errors, + bool(metadata_count and metadata_count > 0), + "native-client-telemetry hook recipes summary.metadata_only_recipe_count must be >0", + ) + add_error( + errors, + privacy.get("raw_content_allowed") is False, + "native-client-telemetry hook recipes privacy_contract.raw_content_allowed must be false", + ) + add_error(errors, bool(recipe_rows), "native-client-telemetry hook recipes rows must not be empty") + if recipe_count is not None: + add_error(errors, len(recipe_rows) == recipe_count, "native-client-telemetry hook recipes length must equal summary.recipe_count") + if metadata_count is not None: + add_error( + errors, + sum(1 for item in recipe_rows if item.get("metadata_only") is True) == metadata_count, + "native-client-telemetry metadata_only recipe rows must match summary.metadata_only_recipe_count", + ) + add_error( + errors, + all(item.get("source") == "external" for item in recipe_rows), + "native-client-telemetry hook recipe rows must use source external", + ) + add_error( + errors, + all(item.get("native_auto_capture") is False for item in recipe_rows), + "native-client-telemetry hook recipe rows must not claim native_auto_capture", + ) + + +def validate_native_telemetry_report(adoption: dict[str, Any], recipes: dict[str, Any], errors: list[str]) -> None: + if adoption: + validate_adoption_report(adoption, errors) + if recipes: + validate_recipe_report(recipes, errors) diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 7c9794d5..929879fc 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(ROOT / "scripts")) from world_class_evidence_contract import validate_payload # noqa: E402 from world_class_human_fixtures import assert_human_contract_artifact_validation # noqa: E402 +from world_class_native_telemetry_fixtures import assert_native_telemetry_contract_artifact_validation # noqa: E402 def sha256_file(path: Path) -> str: @@ -296,53 +297,6 @@ def native_permission_submission(skill_root: Path) -> dict: ) -def write_native_telemetry_artifacts(skill_root: Path, *, complete: bool) -> None: - write_json( - skill_root / "reports" / "adoption_drift_report.json", - { - "schema_version": "2.0", - "ok": True, - "privacy_contract": { - "raw_content_allowed": False, - "raw_event_log_packaged": False, - }, - "summary": { - "source_types": {"external": 1 if complete else 0}, - "adoption_sample_count": 1 if complete else 0, - }, - }, - ) - write_json( - skill_root / "reports" / "telemetry_hook_recipes.json", - { - "schema_version": "1.0", - "ok": True, - "privacy_contract": {"raw_content_allowed": False}, - "summary": {"metadata_only_recipe_count": 5}, - }, - ) - - -def native_telemetry_submission(skill_root: Path) -> dict: - return external_submission( - skill_root, - evidence_key="native-client-telemetry", - source_type="native-client-telemetry", - submitted_by="Yao client integrator", - artifact_paths=("reports/adoption_drift_report.json", "reports/telemetry_hook_recipes.json"), - artifact_kinds={ - "reports/adoption_drift_report.json": "adoption-drift-report", - "reports/telemetry_hook_recipes.json": "hook-recipes", - }, - provenance={ - "client": "Chrome extension production build", - "native_host_manifest": "/Users/laoyao/.config/chrome/native-hosts/yao-meta-skill.json", - "event_source": "external", - "metadata_only": True, - }, - ) - - def write_json(path: Path, payload: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") @@ -492,28 +446,6 @@ def assert_external_contract_artifact_validation() -> None: assert permission_invalid["status"] == "fail", permission_invalid assert any("summary.native_enforcement_count must be >0" in error for error in permission_invalid["errors"]), permission_invalid["errors"] - telemetry_entry = {"key": "native-client-telemetry", "category": "external"} - write_native_telemetry_artifacts(skill_root, complete=True) - telemetry_valid = validate_payload( - native_telemetry_submission(skill_root), - telemetry_entry, - path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", - root=skill_root, - template_expected=False, - ) - assert telemetry_valid["status"] == "pass", telemetry_valid - write_native_telemetry_artifacts(skill_root, complete=False) - telemetry_invalid = validate_payload( - native_telemetry_submission(skill_root), - telemetry_entry, - path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", - root=skill_root, - template_expected=False, - ) - assert telemetry_invalid["status"] == "fail", telemetry_invalid - assert any("summary.source_types.external must be >0" in error for error in telemetry_invalid["errors"]), telemetry_invalid["errors"] - assert any("summary.adoption_sample_count must be >0" in error for error in telemetry_invalid["errors"]), telemetry_invalid["errors"] - def main() -> None: shutil.rmtree(TMP, ignore_errors=True) @@ -522,6 +454,7 @@ def main() -> None: default_md = TMP / "world_class_evidence_intake.md" assert_human_contract_artifact_validation() assert_external_contract_artifact_validation() + assert_native_telemetry_contract_artifact_validation() payload = run_intake("--output-json", str(default_json), "--output-md", str(default_md)) assert payload["schema_version"] == "1.0", payload assert payload["ok"] is True, payload diff --git a/tests/world_class_native_telemetry_fixtures.py b/tests/world_class_native_telemetry_fixtures.py new file mode 100644 index 00000000..cc3fb416 --- /dev/null +++ b/tests/world_class_native_telemetry_fixtures.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +import hashlib +import json +from pathlib import Path + +from world_class_evidence_contract import validate_payload + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_world_class_evidence_intake" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def native_telemetry_submission(skill_root: Path) -> dict: + return { + "schema_version": "1.0", + "evidence_key": "native-client-telemetry", + "template_only": False, + "category": "external", + "source_type": "native-client-telemetry", + "submitted_by": "Yao client integrator", + "submitted_at": "2026-06-14", + "summary": "Completed native-client-telemetry evidence for ledger review.", + "artifact_refs": [ + { + "path": "reports/adoption_drift_report.json", + "kind": "adoption-drift-report", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "adoption_drift_report.json"), + }, + { + "path": "reports/telemetry_hook_recipes.json", + "kind": "hook-recipes", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "telemetry_hook_recipes.json"), + }, + ], + "provenance": { + "client": "Chrome extension production build", + "native_host_manifest": "/Users/laoyao/.config/chrome/native-hosts/yao-meta-skill.json", + "event_source": "external", + "metadata_only": True, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + "ledger_reviewer_approved": True, + "ledger_reviewer": "Yao ledger reviewer", + "ledger_reviewed_at": "2026-06-14", + }, + } + + +def write_native_telemetry_artifacts(skill_root: Path, *, complete: bool) -> None: + recent_events = ( + [ + { + "command": "chrome-extension", + "event": "skill_activation", + "skill": "yao-meta-skill", + "source": "external", + "version": "1.1.0", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "timestamp": "2026-06-14T10:00:00Z", + } + ] + if complete + else [] + ) + write_json( + skill_root / "reports" / "adoption_drift_report.json", + { + "schema_version": "2.0", + "ok": True, + "privacy_contract": { + "raw_content_allowed": False, + "raw_event_log_packaged": False, + }, + "summary": { + "source_types": {"external": 1 if complete else 0}, + "adoption_sample_count": 1 if complete else 0, + }, + "adoption_by_skill": [ + { + "skill": "yao-meta-skill", + "events": 1 if complete else 0, + "adoption_events": 1 if complete else 0, + "accepted": 1 if complete else 0, + "edited": 0, + "rejected": 0, + "missed": 0, + "adoption_rate": 100.0 if complete else 0, + } + ], + "recent_events": recent_events, + }, + ) + write_json( + skill_root / "reports" / "telemetry_hook_recipes.json", + { + "schema_version": "1.0", + "ok": True, + "privacy_contract": {"raw_content_allowed": False}, + "summary": {"recipe_count": 1, "metadata_only_recipe_count": 1}, + "recipes": [ + { + "id": "chrome-extension", + "source": "external", + "metadata_only": True, + "native_auto_capture": False, + "event": "skill_activation", + "outcome": "accepted", + } + ], + }, + ) + + +def assert_native_telemetry_contract_artifact_validation() -> None: + entry = {"key": "native-client-telemetry", "category": "external"} + skill_root = TMP / "native_telemetry_contract_root" + write_native_telemetry_artifacts(skill_root, complete=True) + valid = validate_payload( + native_telemetry_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", + root=skill_root, + template_expected=False, + ) + assert valid["status"] == "pass", valid + + forged_adoption = json.loads((skill_root / "reports" / "adoption_drift_report.json").read_text(encoding="utf-8")) + forged_adoption["recent_events"][0]["source"] = "manual" + write_json(skill_root / "reports" / "adoption_drift_report.json", forged_adoption) + forged = validate_payload( + native_telemetry_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", + root=skill_root, + template_expected=False, + ) + assert forged["status"] == "fail", forged + assert any("external event rows must cover summary.source_types.external" in error for error in forged["errors"]), ( + forged["errors"] + ) + + write_native_telemetry_artifacts(skill_root, complete=False) + invalid = validate_payload( + native_telemetry_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", + root=skill_root, + template_expected=False, + ) + assert invalid["status"] == "fail", invalid + assert any("summary.source_types.external must be >0" in error for error in invalid["errors"]), invalid["errors"] + assert any("summary.adoption_sample_count must be >0" in error for error in invalid["errors"]), invalid["errors"]