frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Validate native telemetry evidence rows

Browse Source
This commit is contained in:
yaojingang
2026-06-16 23:37:57 +08:00
parent dfda1fc0c0
commit 03aa628712
18 changed files with 407 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reports/architecture_maintainability.json
+15 -15
View File
@@ -4,10 +4,10 @@
"generated_at": "2026-06-16",
"skill_dir": ".",
"summary": {
"python_file_count": 212,
"script_file_count": 138,
"test_file_count": 74,
"internal_module_count": 54,
"python_file_count": 214,
"script_file_count": 139,
"test_file_count": 75,
"internal_module_count": 55,
"cli_script_count": 86,
"command_handler_count": 68,
"entrypoint_command_handler_count": 18,
@@ -36,13 +36,6 @@
"severity": "pass",
"recommendation": "Split viewer data assembly from HTML section rendering."
},
{
"path": "tests/verify_world_class_evidence_intake.py",
"lines": 677,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/skill_report_model.py",
"lines": 665,
@@ -93,11 +86,11 @@
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/world_class_evidence_contract.py",
"lines": 618,
"kind": "internal-module",
"path": "tests/verify_world_class_evidence_intake.py",
"lines": 610,
"kind": "test",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/render_benchmark_reproducibility.py",
@@ -105,6 +98,13 @@
"kind": "cli-script",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/optimize_description.py",
"lines": 585,
"kind": "cli-script",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
}
],
"watchlist": [],
reports/architecture_maintainability.md
+6 -6
View File
@@ -5,10 +5,10 @@ Generated at: `2026-06-16`
## Summary
- decision: `pass`
- python files: `212`
- scripts: `138`
- tests: `74`
- internal modules: `54`
- python files: `214`
- scripts: `139`
- tests: `75`
- internal modules: `55`
- CLI scripts: `86`
- Yao CLI command handlers: `68`
- entrypoint command handlers: `18`
@@ -35,7 +35,6 @@ No near-threshold files found.
| --- | ---: | --- | --- |
| `tests/verify_yao_cli.py` | `696` | `test` | `pass` |
| `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` |
| `tests/verify_world_class_evidence_intake.py` | `677` | `test` | `pass` |
| `scripts/skill_report_model.py` | `665` | `internal-module` | `pass` |
| `scripts/render_skill_os2_coverage.py` | `649` | `cli-script` | `pass` |
| `scripts/render_review_studio.py` | `647` | `cli-script` | `pass` |
@@ -43,8 +42,9 @@ No near-threshold files found.
| `scripts/review_studio_gates.py` | `643` | `internal-module` | `pass` |
| `scripts/cross_packager.py` | `638` | `cli-script` | `pass` |
| `scripts/build_skill_atlas.py` | `637` | `cli-script` | `pass` |
| `scripts/world_class_evidence_contract.py` | `618` | `internal-module` | `pass` |
| `tests/verify_world_class_evidence_intake.py` | `610` | `test` | `pass` |
| `scripts/render_benchmark_reproducibility.py` | `595` | `cli-script` | `pass` |
| `scripts/optimize_description.py` | `585` | `cli-script` | `pass` |
## Release Rule
reports/context_budget.json
+10 -10
View File
@@ -6,16 +6,16 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 797,
"other_text_tokens": 1067291,
"other_text_tokens": 1069589,
"estimated_initial_load_tokens": 990,
"estimated_total_text_tokens": 1068088,
"deferred_resource_tokens": 494370,
"estimated_total_text_tokens": 1070386,
"deferred_resource_tokens": 495216,
"deferred_resource_warn_threshold": 120000,
"deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 434272,
"file_count": 138
"estimated_tokens": 435118,
"file_count": 139
},
{
"path": "references",
@@ -36,8 +36,8 @@
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 434272,
"file_count": 138
"estimated_tokens": 435118,
"file_count": 139
}
],
"deferred_resource_governance": {
@@ -59,14 +59,14 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 434272,
"file_count": 138,
"estimated_tokens": 435118,
"file_count": 139,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
],
"summary": "Large deferred resources are indexed and backed by evidence."
},
"relevant_file_count": 671,
"relevant_file_count": 675,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 131.3
reports/context_budget.md
+1 -1
View File
@@ -2,7 +2,7 @@
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
| root | `.` | `production` | 1000 | 990 | 797 | 494370 | `governed` | scripts:434272 | 131.3 | - | ok |
| root | `.` | `production` | 1000 | 990 | 797 | 495216 | `governed` | scripts:435118 | 131.3 | - | ok |
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
reports/context_budget_summary.json
+5 -5
View File
@@ -8,12 +8,12 @@
"budget_limit": 1000,
"initial_tokens": 990,
"skill_body_tokens": 797,
"deferred_resource_tokens": 494370,
"deferred_resource_tokens": 495216,
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 434272,
"file_count": 138
"estimated_tokens": 435118,
"file_count": 139
}
],
"deferred_resource_governance": {
@@ -35,8 +35,8 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 434272,
"file_count": 138,
"estimated_tokens": 435118,
"file_count": 139,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
],
reports/python_compatibility.json
+13 -1
View File
@@ -5,7 +5,7 @@
"root": ".",
"summary": {
"target_python": "3.11",
"file_count": 215,
"file_count": 217,
"issue_count": 0,
"syntax_error_count": 0,
"fstring_311_violation_count": 0,
@@ -766,6 +766,12 @@
"issue_count": 0,
"issues": []
},
{
"path": "scripts/world_class_native_telemetry_evidence.py",
"ok": true,
"issue_count": 0,
"issues": []
},
{
"path": "scripts/world_class_preflight_layout.py",
"ok": true,
@@ -1300,6 +1306,12 @@
"issue_count": 0,
"issues": []
},
{
"path": "tests/world_class_native_telemetry_fixtures.py",
"ok": true,
"issue_count": 0,
"issues": []
},
{
"path": "tests/yao_cli_helpers.py",
"ok": true,
reports/python_compatibility.md
+1 -1
View File
@@ -6,7 +6,7 @@ Generated at: `2026-06-16`
- decision: `pass`
- target python: `3.11`
- files scanned: `215`
- files scanned: `217`
- issues: `0`
- syntax errors: `0`
- f-string 3.11 violations: `0`
reports/security_trust_report.json
+24 -6
View File
@@ -2,9 +2,9 @@
"ok": true,
"skill_dir": ".",
"summary": {
"scanned_files": 229,
"script_count": 138,
"internal_module_count": 52,
"scanned_files": 230,
"script_count": 139,
"internal_module_count": 53,
"secret_findings": 0,
"dependency_files": [
"requirements-ci.txt"
@@ -22,8 +22,8 @@
"help_smoke_failed_count": 0,
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 229,
"package_sha256": "2bc5e8214814c344534e5155301cd3f86e4ceee351313fb3a538d6fd9d400b75"
"package_hash_file_count": 230,
"package_sha256": "79e555f12cb95ad51753c69fdb7fce367f3a394c3e55c075ab4754feb6d3200b"
},
"failures": [],
"warnings": [],
@@ -1740,6 +1740,20 @@
"network_urls": [],
"network_hosts": []
},
{
"path": "scripts/world_class_native_telemetry_evidence.py",
"interface": "internal-module",
"interface_declared": true,
"interface_reason": "Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows.",
"has_argparse": false,
"has_main_guard": false,
"uses_input": false,
"uses_network": false,
"uses_file_write": false,
"uses_subprocess": false,
"network_urls": [],
"network_hosts": []
},
{
"path": "scripts/world_class_preflight_layout.py",
"interface": "internal-module",
@@ -2013,7 +2027,7 @@
"checked_count": 86,
"passed_count": 86,
"failed_count": 0,
"skipped_count": 52,
"skipped_count": 53,
"failed_scripts": [],
"results": [
{
@@ -3022,6 +3036,10 @@
"path": "scripts/world_class_native_permission_evidence.py",
"reason": "internal module"
},
{
"path": "scripts/world_class_native_telemetry_evidence.py",
"reason": "internal module"
},
{
"path": "scripts/world_class_preflight_layout.py",
"reason": "internal module"
reports/security_trust_report.md
+6 -5
View File
@@ -1,9 +1,9 @@
# Security Trust Report
- OK: `True`
- Scanned files: `229`
- Scripts: `138`
- Internal script modules: `52`
- Scanned files: `230`
- Scripts: `139`
- Internal script modules: `53`
- Secret findings: `0`
- Network-capable scripts: `3`
- Network policy covered scripts: `3`
@@ -15,8 +15,8 @@
- CLI help smoke failures: `0`
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `229`
- Package SHA256: `2bc5e8214814c344534e5155301cd3f86e4ceee351313fb3a538d6fd9d400b75`
- Package hash files: `230`
- Package SHA256: `79e555f12cb95ad51753c69fdb7fce367f3a394c3e55c075ab4754feb6d3200b`
## Failures
@@ -183,6 +183,7 @@
| scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. |
| scripts/world_class_human_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows. |
| scripts/world_class_native_permission_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows. |
| scripts/world_class_native_telemetry_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows. |
| scripts/world_class_preflight_layout.py | internal-module | True | False | False | False | False | False | False | Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly. |
| scripts/world_class_provider_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows. |
| scripts/world_class_source_checks.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to keep source-evidence readiness checks consistent. |
reports/skill-os-2-review.md
+3 -2
View File
@@ -23,6 +23,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
- World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `<evidence-key>.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields.
- World-Class Provider Evidence Guard v0 so provider holdout submissions must reconcile `summary` counts with `runs` rows and include at least one passing model run whose provider, model, timing, non-estimated usage, and output hash match the submitted provenance.
- World-Class Native Permission Evidence Guard v0 so native-permission submissions must reconcile runtime probe summary counts with target rows, require at least one native-enforced target row, and keep installer permission checks failure-free.
- World-Class Native Telemetry Evidence Guard v0 so native-client-telemetry submissions must reconcile adoption summary counts with external metadata event rows and keep hook recipes metadata-only without claiming native auto-capture.
- World-Class Submission Matrix v0 so the submission kit exposes a single operator-facing matrix for draft status, artifact readiness, source-check blockers, and the next action without counting matrix rows as completion evidence.
- World-Class Submission Artifact Roles v0 so submission kits distinguish `submission-ref` rows that belong in `artifact_refs` from supporting audit assets, reducing operator ambiguity without accepting evidence.
- World-Class Submission Kit Rendering v0 so Markdown and HTML kit presentation lives in a dedicated internal renderer, keeping the CLI focused on evidence assembly and file emission.
@@ -68,7 +69,7 @@ This is still not the final world-class state. Target-native behavior contracts
| Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed |
| World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed |
| World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed |
| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, provider run-row validation, native permission target-row validation, and nested raw-field rejection | v0 landed |
| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/world_class_native_telemetry_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, provider run-row validation, native permission target-row validation, native telemetry event-row validation, and nested raw-field rejection | v0 landed |
| World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed |
| Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed |
| Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed |
@@ -129,7 +130,7 @@ Next move: add real client or installer permission enforcement integration.
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden |
| Runtime Conformance | `5 / 5` targets passing |
| Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts |
| Trust | `0` secret findings, `1` pinned dependency file, `52` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `138` scripts, source-contract hash scope explicit |
| Trust | `0` secret findings, `1` pinned dependency file, `53` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `139` scripts, source-contract hash scope explicit |
| Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired |
| Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility |
| Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility |
reports/skill_os2_audit.json
+1 -1
View File
@@ -197,7 +197,7 @@
"key": "trust-security",
"label": "Trust Security",
"status": "pass",
"current": "secrets 0; scripts 138; help failures 0",
"current": "secrets 0; scripts 139; help failures 0",
"target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable",
"evidence": [
{
reports/skill_os2_audit.md
+1 -1
View File
@@ -23,7 +23,7 @@ Generated at: `2026-06-16`
| Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. |
| Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
| Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. |
| Trust Security | pass | secrets 0; scripts 138; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
| Trust Security | pass | secrets 0; scripts 139; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
| Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. |
| Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. |
| Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. |
reports/skill_os2_coverage.json
+1 -1
View File
@@ -130,7 +130,7 @@
"label": "Trust Security",
"status": "pass",
"objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.",
"current": "138 scripts; secrets 0; help failures 0",
"current": "139 scripts; secrets 0; help failures 0",
"command": "python3 scripts/yao.py trust .",
"test": "python3 tests/verify_trust_check.py",
"evidence": [
reports/skill_os2_coverage.md
+1 -1
View File
@@ -26,7 +26,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
| Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` |
| Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` |
| Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` |
| Trust Security | `pass` | 138 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
| Trust Security | `pass` | 139 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
| Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` |
| Registry Distribution | `pass` | archive entries 658; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
scripts/world_class_evidence_contract.py
+2 -60
View File
@@ -8,6 +8,7 @@ from typing import Any
from world_class_human_evidence import validate_human_adjudication_report
from world_class_native_permission_evidence import validate_native_permission_report
from world_class_native_telemetry_evidence import validate_native_telemetry_report
from world_class_provider_evidence import validate_provider_execution_report
@@ -369,25 +370,6 @@ def artifact_ref_path_map(payload: dict[str, Any], root: Path) -> dict[str, Path
return paths
def real_int(value: Any) -> int | None:
return value if isinstance(value, int) and not isinstance(value, bool) else None
def summary(payload: dict[str, Any]) -> dict[str, Any]:
value = payload.get("summary", {})
return value if isinstance(value, dict) else {}
def source_types(payload: dict[str, Any]) -> dict[str, Any]:
value = summary(payload).get("source_types", {})
return value if isinstance(value, dict) else {}
def privacy_contract(payload: dict[str, Any]) -> dict[str, Any]:
value = payload.get("privacy_contract", {})
return value if isinstance(value, dict) else {}
def validate_provider_holdout_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
paths = artifact_ref_path_map(payload, root)
execution = load_json(paths.get("reports/output_execution_runs.json", root / "__missing__"))
@@ -419,47 +401,7 @@ def validate_native_client_telemetry_artifacts(payload: dict[str, Any], errors:
paths = artifact_ref_path_map(payload, root)
adoption = load_json(paths.get("reports/adoption_drift_report.json", root / "__missing__"))
recipes = load_json(paths.get("reports/telemetry_hook_recipes.json", root / "__missing__"))
if adoption:
adoption_summary = summary(adoption)
adoption_privacy = privacy_contract(adoption)
add_error(errors, adoption.get("ok") is True, "native-client-telemetry adoption drift report ok must be true")
add_error(
errors,
bool(real_int(source_types(adoption).get("external")) and source_types(adoption)["external"] > 0),
"native-client-telemetry adoption drift summary.source_types.external must be >0",
)
add_error(
errors,
bool(real_int(adoption_summary.get("adoption_sample_count")) and adoption_summary["adoption_sample_count"] > 0),
"native-client-telemetry adoption drift summary.adoption_sample_count must be >0",
)
add_error(
errors,
adoption_privacy.get("raw_content_allowed") is False,
"native-client-telemetry adoption drift privacy_contract.raw_content_allowed must be false",
)
add_error(
errors,
adoption_privacy.get("raw_event_log_packaged") is False,
"native-client-telemetry adoption drift privacy_contract.raw_event_log_packaged must be false",
)
if recipes:
recipes_summary = summary(recipes)
recipes_privacy = privacy_contract(recipes)
add_error(errors, recipes.get("ok") is True, "native-client-telemetry hook recipes report ok must be true")
add_error(
errors,
bool(
real_int(recipes_summary.get("metadata_only_recipe_count"))
and recipes_summary["metadata_only_recipe_count"] > 0
),
"native-client-telemetry hook recipes summary.metadata_only_recipe_count must be >0",
)
add_error(
errors,
recipes_privacy.get("raw_content_allowed") is False,
"native-client-telemetry hook recipes privacy_contract.raw_content_allowed must be false",
)
validate_native_telemetry_report(adoption, recipes, errors)
def validate_real_artifact_payloads(payload: dict[str, Any], errors: list[str], root: Path, template_expected: bool) -> None:
scripts/world_class_native_telemetry_evidence.py
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
from typing import Any
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows."
ADOPTION_EVENTS = {"skill_activation", "skill_output"}
def add_error(errors: list[str], condition: bool, message: str) -> None:
if not condition:
errors.append(message)
def real_int(value: Any) -> int | None:
return value if isinstance(value, int) and not isinstance(value, bool) else None
def object_list(value: Any) -> list[dict[str, Any]]:
return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else []
def source_types(payload: dict[str, Any]) -> dict[str, Any]:
summary = payload.get("summary", {}) if isinstance(payload.get("summary"), dict) else {}
value = summary.get("source_types", {})
return value if isinstance(value, dict) else {}
def validate_adoption_report(adoption: dict[str, Any], errors: list[str]) -> None:
summary = adoption.get("summary", {}) if isinstance(adoption.get("summary"), dict) else {}
privacy = adoption.get("privacy_contract", {}) if isinstance(adoption.get("privacy_contract"), dict) else {}
events = object_list(adoption.get("recent_events", []))
adoption_rows = object_list(adoption.get("adoption_by_skill", []))
external_summary_count = real_int(source_types(adoption).get("external"))
adoption_sample_count = real_int(summary.get("adoption_sample_count"))
add_error(errors, adoption.get("ok") is True, "native-client-telemetry adoption drift report ok must be true")
add_error(
errors,
bool(external_summary_count and external_summary_count > 0),
"native-client-telemetry adoption drift summary.source_types.external must be >0",
)
add_error(
errors,
bool(adoption_sample_count and adoption_sample_count > 0),
"native-client-telemetry adoption drift summary.adoption_sample_count must be >0",
)
add_error(
errors,
privacy.get("raw_content_allowed") is False,
"native-client-telemetry adoption drift privacy_contract.raw_content_allowed must be false",
)
add_error(
errors,
privacy.get("raw_event_log_packaged") is False,
"native-client-telemetry adoption drift privacy_contract.raw_event_log_packaged must be false",
)
add_error(errors, bool(events), "native-client-telemetry adoption drift recent_events must not be empty")
external_events = [item for item in events if item.get("source") == "external"]
external_adoption_events = [item for item in external_events if item.get("event") in ADOPTION_EVENTS]
add_error(
errors,
bool(external_summary_count and len(external_events) >= external_summary_count),
"native-client-telemetry external event rows must cover summary.source_types.external",
)
add_error(
errors,
bool(external_adoption_events),
"native-client-telemetry must include at least one external adoption event row",
)
add_error(
errors,
bool(
adoption_sample_count
and len([item for item in events if item.get("event") in ADOPTION_EVENTS]) >= adoption_sample_count
),
"native-client-telemetry adoption event rows must cover summary.adoption_sample_count",
)
add_error(
errors,
any((count := real_int(row.get("adoption_events"))) is not None and count > 0 for row in adoption_rows),
"native-client-telemetry adoption_by_skill must include at least one adoption event",
)
def validate_recipe_report(recipes: dict[str, Any], errors: list[str]) -> None:
summary = recipes.get("summary", {}) if isinstance(recipes.get("summary"), dict) else {}
privacy = recipes.get("privacy_contract", {}) if isinstance(recipes.get("privacy_contract"), dict) else {}
recipe_rows = object_list(recipes.get("recipes", []))
recipe_count = real_int(summary.get("recipe_count"))
metadata_count = real_int(summary.get("metadata_only_recipe_count"))
add_error(errors, recipes.get("ok") is True, "native-client-telemetry hook recipes report ok must be true")
add_error(
errors,
bool(metadata_count and metadata_count > 0),
"native-client-telemetry hook recipes summary.metadata_only_recipe_count must be >0",
)
add_error(
errors,
privacy.get("raw_content_allowed") is False,
"native-client-telemetry hook recipes privacy_contract.raw_content_allowed must be false",
)
add_error(errors, bool(recipe_rows), "native-client-telemetry hook recipes rows must not be empty")
if recipe_count is not None:
add_error(errors, len(recipe_rows) == recipe_count, "native-client-telemetry hook recipes length must equal summary.recipe_count")
if metadata_count is not None:
add_error(
errors,
sum(1 for item in recipe_rows if item.get("metadata_only") is True) == metadata_count,
"native-client-telemetry metadata_only recipe rows must match summary.metadata_only_recipe_count",
)
add_error(
errors,
all(item.get("source") == "external" for item in recipe_rows),
"native-client-telemetry hook recipe rows must use source external",
)
add_error(
errors,
all(item.get("native_auto_capture") is False for item in recipe_rows),
"native-client-telemetry hook recipe rows must not claim native_auto_capture",
)
def validate_native_telemetry_report(adoption: dict[str, Any], recipes: dict[str, Any], errors: list[str]) -> None:
if adoption:
validate_adoption_report(adoption, errors)
if recipes:
validate_recipe_report(recipes, errors)
tests/verify_world_class_evidence_intake.py
+2 -69
View File
@@ -14,6 +14,7 @@ sys.path.insert(0, str(ROOT / "scripts"))
from world_class_evidence_contract import validate_payload # noqa: E402
from world_class_human_fixtures import assert_human_contract_artifact_validation # noqa: E402
from world_class_native_telemetry_fixtures import assert_native_telemetry_contract_artifact_validation # noqa: E402
def sha256_file(path: Path) -> str:
@@ -296,53 +297,6 @@ def native_permission_submission(skill_root: Path) -> dict:
)
def write_native_telemetry_artifacts(skill_root: Path, *, complete: bool) -> None:
write_json(
skill_root / "reports" / "adoption_drift_report.json",
{
"schema_version": "2.0",
"ok": True,
"privacy_contract": {
"raw_content_allowed": False,
"raw_event_log_packaged": False,
},
"summary": {
"source_types": {"external": 1 if complete else 0},
"adoption_sample_count": 1 if complete else 0,
},
},
)
write_json(
skill_root / "reports" / "telemetry_hook_recipes.json",
{
"schema_version": "1.0",
"ok": True,
"privacy_contract": {"raw_content_allowed": False},
"summary": {"metadata_only_recipe_count": 5},
},
)
def native_telemetry_submission(skill_root: Path) -> dict:
return external_submission(
skill_root,
evidence_key="native-client-telemetry",
source_type="native-client-telemetry",
submitted_by="Yao client integrator",
artifact_paths=("reports/adoption_drift_report.json", "reports/telemetry_hook_recipes.json"),
artifact_kinds={
"reports/adoption_drift_report.json": "adoption-drift-report",
"reports/telemetry_hook_recipes.json": "hook-recipes",
},
provenance={
"client": "Chrome extension production build",
"native_host_manifest": "/Users/laoyao/.config/chrome/native-hosts/yao-meta-skill.json",
"event_source": "external",
"metadata_only": True,
},
)
def write_json(path: Path, payload: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
@@ -492,28 +446,6 @@ def assert_external_contract_artifact_validation() -> None:
assert permission_invalid["status"] == "fail", permission_invalid
assert any("summary.native_enforcement_count must be >0" in error for error in permission_invalid["errors"]), permission_invalid["errors"]
telemetry_entry = {"key": "native-client-telemetry", "category": "external"}
write_native_telemetry_artifacts(skill_root, complete=True)
telemetry_valid = validate_payload(
native_telemetry_submission(skill_root),
telemetry_entry,
path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json",
root=skill_root,
template_expected=False,
)
assert telemetry_valid["status"] == "pass", telemetry_valid
write_native_telemetry_artifacts(skill_root, complete=False)
telemetry_invalid = validate_payload(
native_telemetry_submission(skill_root),
telemetry_entry,
path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json",
root=skill_root,
template_expected=False,
)
assert telemetry_invalid["status"] == "fail", telemetry_invalid
assert any("summary.source_types.external must be >0" in error for error in telemetry_invalid["errors"]), telemetry_invalid["errors"]
assert any("summary.adoption_sample_count must be >0" in error for error in telemetry_invalid["errors"]), telemetry_invalid["errors"]
def main() -> None:
shutil.rmtree(TMP, ignore_errors=True)
@@ -522,6 +454,7 @@ def main() -> None:
default_md = TMP / "world_class_evidence_intake.md"
assert_human_contract_artifact_validation()
assert_external_contract_artifact_validation()
assert_native_telemetry_contract_artifact_validation()
payload = run_intake("--output-json", str(default_json), "--output-md", str(default_md))
assert payload["schema_version"] == "1.0", payload
assert payload["ok"] is True, payload
tests/world_class_native_telemetry_fixtures.py
+185
View File
@@ -0,0 +1,185 @@
#!/usr/bin/env python3
import hashlib
import json
from pathlib import Path
from world_class_evidence_contract import validate_payload
ROOT = Path(__file__).resolve().parent.parent
TMP = ROOT / "tests" / "tmp_world_class_evidence_intake"
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def write_json(path: Path, payload: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def native_telemetry_submission(skill_root: Path) -> dict:
return {
"schema_version": "1.0",
"evidence_key": "native-client-telemetry",
"template_only": False,
"category": "external",
"source_type": "native-client-telemetry",
"submitted_by": "Yao client integrator",
"submitted_at": "2026-06-14",
"summary": "Completed native-client-telemetry evidence for ledger review.",
"artifact_refs": [
{
"path": "reports/adoption_drift_report.json",
"kind": "adoption-drift-report",
"contains_raw_content": False,
"sha256": sha256_file(skill_root / "reports" / "adoption_drift_report.json"),
},
{
"path": "reports/telemetry_hook_recipes.json",
"kind": "hook-recipes",
"contains_raw_content": False,
"sha256": sha256_file(skill_root / "reports" / "telemetry_hook_recipes.json"),
},
],
"provenance": {
"client": "Chrome extension production build",
"native_host_manifest": "/Users/laoyao/.config/chrome/native-hosts/yao-meta-skill.json",
"event_source": "external",
"metadata_only": True,
},
"privacy": {
"raw_user_content_included": False,
"raw_provider_prompt_included": False,
"credentials_included": False,
"secrets_included": False,
},
"anti_overclaim": {
"planned_work_counts_as_evidence": False,
"metadata_fallback_counts_as_native_enforcement": False,
"pending_review_counts_as_human_decision": False,
"local_command_runner_counts_as_provider_model": False,
},
"attestation": {
"real_external_or_human_evidence": True,
"reviewer_or_operator_identity_present": True,
"artifact_refs_reviewed": True,
"privacy_contract_satisfied": True,
"ledger_reviewer_approved": True,
"ledger_reviewer": "Yao ledger reviewer",
"ledger_reviewed_at": "2026-06-14",
},
}
def write_native_telemetry_artifacts(skill_root: Path, *, complete: bool) -> None:
recent_events = (
[
{
"command": "chrome-extension",
"event": "skill_activation",
"skill": "yao-meta-skill",
"source": "external",
"version": "1.1.0",
"activation_type": "explicit",
"outcome": "accepted",
"failure_type": "none",
"timestamp": "2026-06-14T10:00:00Z",
}
]
if complete
else []
)
write_json(
skill_root / "reports" / "adoption_drift_report.json",
{
"schema_version": "2.0",
"ok": True,
"privacy_contract": {
"raw_content_allowed": False,
"raw_event_log_packaged": False,
},
"summary": {
"source_types": {"external": 1 if complete else 0},
"adoption_sample_count": 1 if complete else 0,
},
"adoption_by_skill": [
{
"skill": "yao-meta-skill",
"events": 1 if complete else 0,
"adoption_events": 1 if complete else 0,
"accepted": 1 if complete else 0,
"edited": 0,
"rejected": 0,
"missed": 0,
"adoption_rate": 100.0 if complete else 0,
}
],
"recent_events": recent_events,
},
)
write_json(
skill_root / "reports" / "telemetry_hook_recipes.json",
{
"schema_version": "1.0",
"ok": True,
"privacy_contract": {"raw_content_allowed": False},
"summary": {"recipe_count": 1, "metadata_only_recipe_count": 1},
"recipes": [
{
"id": "chrome-extension",
"source": "external",
"metadata_only": True,
"native_auto_capture": False,
"event": "skill_activation",
"outcome": "accepted",
}
],
},
)
def assert_native_telemetry_contract_artifact_validation() -> None:
entry = {"key": "native-client-telemetry", "category": "external"}
skill_root = TMP / "native_telemetry_contract_root"
write_native_telemetry_artifacts(skill_root, complete=True)
valid = validate_payload(
native_telemetry_submission(skill_root),
entry,
path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json",
root=skill_root,
template_expected=False,
)
assert valid["status"] == "pass", valid
forged_adoption = json.loads((skill_root / "reports" / "adoption_drift_report.json").read_text(encoding="utf-8"))
forged_adoption["recent_events"][0]["source"] = "manual"
write_json(skill_root / "reports" / "adoption_drift_report.json", forged_adoption)
forged = validate_payload(
native_telemetry_submission(skill_root),
entry,
path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json",
root=skill_root,
template_expected=False,
)
assert forged["status"] == "fail", forged
assert any("external event rows must cover summary.source_types.external" in error for error in forged["errors"]), (
forged["errors"]
)
write_native_telemetry_artifacts(skill_root, complete=False)
invalid = validate_payload(
native_telemetry_submission(skill_root),
entry,
path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json",
root=skill_root,
template_expected=False,
)
assert invalid["status"] == "fail", invalid
assert any("summary.source_types.external must be >0" in error for error in invalid["errors"]), invalid["errors"]
assert any("summary.adoption_sample_count must be >0" in error for error in invalid["errors"]), invalid["errors"]