Files
yao-meta-skill/reports/skill_os2_coverage.json
T
2026-06-16 03:12:10 +08:00

757 lines
23 KiB
JSON

{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"skill_dir": ".",
"summary": {
"item_count": 21,
"module_count": 8,
"recommended_pr_count": 13,
"pass_count": 21,
"warn_count": 0,
"missing_count": 0,
"extension_track_count": 2,
"extension_partial_count": 0,
"extension_planned_count": 0,
"extension_covered_count": 2,
"adaptive_extension_ready": true,
"local_blueprint_ready": true,
"public_world_class_ready": false,
"world_class_evidence_pending_count": 4,
"decision": "local-blueprint-covered-evidence-pending"
},
"status_counts": {
"pass": 21,
"warn": 0,
"missing": 0
},
"extension_status_counts": {
"covered": 2
},
"modules": [
{
"key": "skill-ir",
"category": "core-module",
"label": "Skill IR",
"status": "pass",
"objective": "Platform-neutral capability contract exists before platform-specific packaging.",
"current": "schema 2.0.0; targets 5",
"command": "python3 scripts/yao.py skill-ir .",
"test": "python3 tests/verify_skill_ir.py",
"evidence": [
{
"path": "skill-ir/schema.json",
"exists": true
},
{
"path": "skill-ir/examples/yao-meta-skill.json",
"exists": true
},
{
"path": "scripts/export_skill_ir.py",
"exists": true
},
{
"path": "tests/verify_skill_ir.py",
"exists": true
}
],
"next_action": "Keep all target packages compiled from IR rather than hand-maintained per target."
},
{
"key": "output-eval-lab",
"category": "core-module",
"label": "Output Eval Lab",
"status": "pass",
"objective": "with-skill/baseline output quality is measured with assertions, execution evidence, and blind review packs.",
"current": "5 cases; delta 100.0; execution 10",
"command": "python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .",
"test": "python3 tests/verify_output_eval_lab.py",
"evidence": [
{
"path": "evals/output/schema.json",
"exists": true
},
{
"path": "evals/output/cases.jsonl",
"exists": true
},
{
"path": "scripts/run_output_eval.py",
"exists": true
},
{
"path": "scripts/run_output_execution.py",
"exists": true
},
{
"path": "reports/output_quality_scorecard.json",
"exists": true
},
{
"path": "tests/verify_output_eval_lab.py",
"exists": true
}
],
"next_action": "Add more real-file and adversarial holdout cases as adoption data grows."
},
{
"key": "runtime-conformance",
"category": "core-module",
"label": "Runtime Conformance",
"status": "pass",
"objective": "Target packages can be consumed by OpenAI, Claude, Agent Skills, VS Code, and generic targets.",
"current": "5/5 targets pass",
"command": "python3 scripts/yao.py conformance .",
"test": "python3 tests/verify_conformance_suite.py",
"evidence": [
{
"path": "runtime/conformance/schema.json",
"exists": true
},
{
"path": "scripts/run_conformance_suite.py",
"exists": true
},
{
"path": "reports/conformance_matrix.json",
"exists": true
},
{
"path": "tests/verify_conformance_suite.py",
"exists": true
}
],
"next_action": "Keep conformance cases aligned with current platform metadata rules."
},
{
"key": "trust-security",
"category": "core-module",
"label": "Trust Security",
"status": "pass",
"objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.",
"current": "114 scripts; secrets 0; help failures 0",
"command": "python3 scripts/yao.py trust .",
"test": "python3 tests/verify_trust_check.py",
"evidence": [
{
"path": "scripts/trust_check.py",
"exists": true
},
{
"path": "security/trust_policy.md",
"exists": true
},
{
"path": "security/script_policy.md",
"exists": true
},
{
"path": "security/permission_policy.json",
"exists": true
},
{
"path": "reports/security_trust_report.json",
"exists": true
},
{
"path": "tests/verify_trust_check.py",
"exists": true
}
],
"next_action": "Keep high-permission approvals scoped, expiring, and mapped to target enforcement."
},
{
"key": "skill-atlas",
"category": "core-module",
"label": "Skill Atlas",
"status": "pass",
"objective": "Team skill portfolio reveals route collisions, stale ownership, dependency graph, and no-route opportunities.",
"current": "12 scanned skills; actionable collisions 0",
"command": "python3 scripts/yao.py skill-atlas --workspace-root .",
"test": "python3 tests/verify_skill_atlas.py",
"evidence": [
{
"path": "scripts/build_skill_atlas.py",
"exists": true
},
{
"path": "skill_atlas/catalog.json",
"exists": true
},
{
"path": "skill_atlas/route_overlap_matrix.csv",
"exists": true
},
{
"path": "skill_atlas/dependency_graph.json",
"exists": true
},
{
"path": "reports/skill_atlas.json",
"exists": true
},
{
"path": "tests/verify_skill_atlas.py",
"exists": true
}
],
"next_action": "Use real telemetry to rank stale or drifting skills by impact, not only by static metadata."
},
{
"key": "registry-distribution",
"category": "core-module",
"label": "Registry Distribution",
"status": "pass",
"objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.",
"current": "archive entries 626; install failures 0",
"command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .",
"test": "python3 tests/verify_registry_audit.py",
"evidence": [
{
"path": "registry/index.schema.json",
"exists": true
},
{
"path": "registry/package.schema.json",
"exists": true
},
{
"path": "registry/packages/yao-meta-skill.json",
"exists": true
},
{
"path": "scripts/registry_audit.py",
"exists": true
},
{
"path": "reports/package_verification.json",
"exists": true
},
{
"path": "reports/install_simulation.json",
"exists": true
},
{
"path": "tests/verify_registry_audit.py",
"exists": true
}
],
"next_action": "Regenerate registry metadata after package verification so source and archive checksums stay aligned."
},
{
"key": "review-studio",
"category": "core-module",
"label": "Review Studio",
"status": "pass",
"objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.",
"current": "16 gates; decision review; warnings 3",
"command": "python3 scripts/yao.py review-studio .",
"test": "python3 tests/verify_review_studio.py",
"evidence": [
{
"path": "scripts/render_review_studio.py",
"exists": true
},
{
"path": "reports/review-studio.html",
"exists": true
},
{
"path": "reports/review-studio.json",
"exists": true
},
{
"path": "tests/verify_review_studio.py",
"exists": true
}
],
"next_action": "Close pending human and external evidence before claiming full release readiness."
},
{
"key": "telemetry-drift",
"category": "core-module",
"label": "Telemetry Drift",
"status": "pass",
"objective": "Real usage feedback is captured as metadata-only local-first drift signals.",
"current": "events 1; recipes 5; risk low",
"command": "python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .",
"test": "python3 tests/verify_telemetry_hooks.py",
"evidence": [
{
"path": "scripts/emit_telemetry_event.py",
"exists": true
},
{
"path": "scripts/import_telemetry_events.py",
"exists": true
},
{
"path": "scripts/telemetry_native_host.py",
"exists": true
},
{
"path": "reports/adoption_drift_report.json",
"exists": true
},
{
"path": "reports/telemetry_hook_recipes.json",
"exists": true
},
{
"path": "tests/verify_telemetry_hooks.py",
"exists": true
}
],
"next_action": "Install a real client and import production metadata-only events into the local drift loop."
}
],
"recommended_prs": [
{
"key": "benchmark-methodology",
"category": "recommended-pr",
"label": "Benchmark Methodology",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "25 required artifacts checked",
"command": "make ci-test",
"test": "tests/verify_benchmark_reproducibility.py",
"evidence": [
{
"path": "reports/benchmark_methodology.md",
"exists": true
},
{
"path": "reports/benchmark_reproducibility.json",
"exists": true
},
{
"path": "tests/verify_benchmark_reproducibility.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "output-eval-schema",
"category": "recommended-pr",
"label": "Output Eval Schema",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "5 output cases",
"command": "make ci-test",
"test": "tests/verify_output_eval_lab.py",
"evidence": [
{
"path": "evals/output/schema.json",
"exists": true
},
{
"path": "evals/output/cases.jsonl",
"exists": true
},
{
"path": "tests/verify_output_eval_lab.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "output-eval-runner",
"category": "recommended-pr",
"label": "Output Eval Runner",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "delta 100.0",
"command": "make ci-test",
"test": "tests/verify_output_eval_lab.py",
"evidence": [
{
"path": "scripts/run_output_eval.py",
"exists": true
},
{
"path": "reports/output_quality_scorecard.json",
"exists": true
},
{
"path": "tests/verify_output_eval_lab.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "output-quality-scorecard",
"category": "recommended-pr",
"label": "Output Quality Scorecard",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "gate pass True",
"command": "make ci-test",
"test": "tests/verify_output_eval_lab.py",
"evidence": [
{
"path": "reports/output_quality_scorecard.md",
"exists": true
},
{
"path": "reports/output_quality_scorecard.json",
"exists": true
},
{
"path": "tests/verify_output_eval_lab.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "skill-ir-v0",
"category": "recommended-pr",
"label": "Skill IR V0",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "schema 2.0.0",
"command": "make ci-test",
"test": "tests/verify_skill_ir.py",
"evidence": [
{
"path": "skill-ir/schema.json",
"exists": true
},
{
"path": "skill-ir/examples/yao-meta-skill.json",
"exists": true
},
{
"path": "tests/verify_skill_ir.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "compiler-refactor",
"category": "recommended-pr",
"label": "Compiler Refactor",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "5/5 compiled targets",
"command": "make ci-test",
"test": "tests/verify_compile_skill.py",
"evidence": [
{
"path": "scripts/compile_skill.py",
"exists": true
},
{
"path": "reports/compiled_targets.json",
"exists": true
},
{
"path": "tests/verify_compile_skill.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "agent-skills-conformance",
"category": "recommended-pr",
"label": "Agent Skills Conformance",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "agent-skills target present",
"command": "make ci-test",
"test": "tests/verify_conformance_suite.py",
"evidence": [
{
"path": "runtime/conformance/schema.json",
"exists": true
},
{
"path": "reports/conformance_matrix.json",
"exists": true
},
{
"path": "tests/verify_conformance_suite.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "trust-check",
"category": "recommended-pr",
"label": "Trust Check",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "secret findings 0",
"command": "make ci-test",
"test": "tests/verify_trust_check.py",
"evidence": [
{
"path": "scripts/trust_check.py",
"exists": true
},
{
"path": "reports/security_trust_report.json",
"exists": true
},
{
"path": "tests/verify_trust_check.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "skill-atlas-generator",
"category": "recommended-pr",
"label": "Skill Atlas Generator",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "12 scanned skills",
"command": "make ci-test",
"test": "tests/verify_skill_atlas.py",
"evidence": [
{
"path": "scripts/build_skill_atlas.py",
"exists": true
},
{
"path": "reports/skill_atlas.json",
"exists": true
},
{
"path": "tests/verify_skill_atlas.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "registry-package-format",
"category": "recommended-pr",
"label": "Registry Package Format",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "registry ok True",
"command": "make ci-test",
"test": "tests/verify_registry_audit.py",
"evidence": [
{
"path": "registry/package.schema.json",
"exists": true
},
{
"path": "reports/registry_audit.json",
"exists": true
},
{
"path": "tests/verify_registry_audit.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "review-studio-2",
"category": "recommended-pr",
"label": "Review Studio 2.0",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "16 review gates",
"command": "make ci-test",
"test": "tests/verify_review_studio.py",
"evidence": [
{
"path": "scripts/render_review_studio.py",
"exists": true
},
{
"path": "reports/review-studio.html",
"exists": true
},
{
"path": "tests/verify_review_studio.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "migration-v2-docs",
"category": "recommended-pr",
"label": "Migration V2 Docs",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "migration guide present",
"command": "make ci-test",
"test": "docs review",
"evidence": [
{
"path": "docs/migration-v2.md",
"exists": true
},
{
"path": "reports/skill-os-2-review.md",
"exists": true
},
{
"path": "README.md",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "evidence-consistency",
"category": "recommended-pr",
"label": "Evidence Consistency",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "34 consistency checks",
"command": "make ci-test",
"test": "tests/verify_evidence_consistency.py",
"evidence": [
{
"path": "scripts/render_evidence_consistency.py",
"exists": true
},
{
"path": "reports/evidence_consistency.json",
"exists": true
},
{
"path": "tests/verify_evidence_consistency.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
}
],
"reference_extension_tracks": [
{
"key": "skill-interpretation-report",
"label": "Skill Interpretation Report",
"status": "covered",
"objective": "User-facing deep interpretation report explains use cases, triggers, inputs, outputs, workflow, principles, boundaries, quality gates, examples, and next iterations.",
"current": "Skill Overview v2 is canonical and mirrored as first-class skill-interpretation HTML/JSON with schema and tests.",
"target": "Either keep skill-overview as the canonical interpretation report with an explicit contract, or split a dedicated reports/skill-interpretation.* renderer and tests.",
"evidence": [
{
"path": "reports/skill-overview.html",
"exists": true
},
{
"path": "reports/skill-overview.json",
"exists": true
},
{
"path": "scripts/render_skill_overview.py",
"exists": true
},
{
"path": "scripts/render_skill_interpretation.py",
"exists": true
},
{
"path": "schemas/skill-interpretation.schema.json",
"exists": true
},
{
"path": "tests/verify_skill_interpretation.py",
"exists": true
}
],
"next_action": "Keep overview and interpretation contracts in lockstep when report sections, metrics, or layout semantics change."
},
{
"key": "adaptive-self-iteration",
"label": "Adaptive Self-Iteration",
"status": "covered",
"objective": "Local-first preference memory, repeated-signal extraction, adaptation proposals, approval, patch application, regression evidence, and rollback.",
"current": "Full adaptive loop includes proposal, approval, patch application, regression evidence, and rollback metadata.",
"target": "Proposal-only adaptation with explicit input source, redaction, allowlisted write targets, approval ledger, regression report, and rollback plan.",
"evidence": [
{
"path": "references/autonomous-adaptation.md",
"exists": true
},
{
"path": "references/user-memory-policy.md",
"exists": true
},
{
"path": "schemas/adaptation-proposal.schema.json",
"exists": true
},
{
"path": "scripts/summarize_user_signals.py",
"exists": true
},
{
"path": "scripts/propose_adaptation.py",
"exists": true
},
{
"path": "tests/verify_adaptation_safety.py",
"exists": true
},
{
"path": "scripts/apply_adaptation.py",
"exists": true
},
{
"path": "reports/adaptation_approval_ledger.json",
"exists": true
},
{
"path": "reports/adaptation_regression_report.json",
"exists": true
},
{
"path": "reports/user_patterns.json",
"exists": true
},
{
"path": "reports/adaptation_proposals.json",
"exists": true
},
{
"path": "reports/iteration-directions.md",
"exists": true
},
{
"path": "reports/adoption_drift_report.md",
"exists": true
}
],
"next_action": "Start with policy and read-only scan tests; do not read shell history or private logs unless the user provides an explicit source path."
}
],
"next_highest_leverage": [
"Close the four world-class evidence ledger entries with accepted human or external evidence.",
"Keep the first-class skill interpretation report and Skill Overview v2 contract synchronized as the report model evolves.",
"Start adaptive self-iteration as explicit-source, proposal-only, approval-gated work; do not scan private logs by default.",
"Keep the blueprint coverage report in CI so 2.0 plan drift is visible."
],
"source_blueprint": {
"title": "Skill Overview / Skill OS 2.0 upgrade plan",
"core_module_count": 8,
"recommended_pr_count": 13,
"reference_extension_count": 2,
"reference_extensions": [
"Skill interpretation report",
"Adaptive self-iteration with local preference memory and approval gates"
]
},
"artifacts": {
"json": "reports/skill_os2_coverage.json",
"markdown": "reports/skill_os2_coverage.md"
}
}