Files
yao-meta-skill/reports/skill_os2_coverage.json
2026-07-02 11:43:17 +08:00

812 lines
26 KiB
JSON

{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-07-02",
"skill_dir": ".",
"summary": {
"item_count": 21,
"module_count": 8,
"recommended_pr_count": 13,
"pass_count": 21,
"warn_count": 0,
"missing_count": 0,
"extension_track_count": 4,
"extension_partial_count": 0,
"extension_planned_count": 0,
"extension_covered_count": 4,
"adaptive_extension_ready": true,
"local_blueprint_ready": true,
"public_world_class_ready": false,
"world_class_evidence_pending_count": 4,
"decision": "local-blueprint-covered-evidence-pending"
},
"status_counts": {
"pass": 21,
"warn": 0,
"missing": 0
},
"extension_status_counts": {
"covered": 4
},
"modules": [
{
"key": "skill-ir",
"category": "core-module",
"label": "Skill IR",
"status": "pass",
"objective": "Platform-neutral capability contract exists before platform-specific packaging.",
"current": "schema 2.0.0; targets 5",
"command": "python3 scripts/yao.py skill-ir .",
"test": "python3 tests/verify_skill_ir.py",
"evidence": [
{
"path": "skill-ir/schema.json",
"exists": true
},
{
"path": "skill-ir/examples/yao-meta-skill.json",
"exists": true
},
{
"path": "scripts/export_skill_ir.py",
"exists": true
},
{
"path": "tests/verify_skill_ir.py",
"exists": true
}
],
"next_action": "Keep all target packages compiled from IR rather than hand-maintained per target."
},
{
"key": "output-eval-lab",
"category": "core-module",
"label": "Output Eval Lab",
"status": "pass",
"objective": "with-skill/baseline output quality is measured with assertions, execution evidence, and blind review packs.",
"current": "5 cases; delta 100.0; execution 10",
"command": "python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .",
"test": "python3 tests/verify_output_eval_lab.py",
"evidence": [
{
"path": "evals/output/schema.json",
"exists": true
},
{
"path": "evals/output/cases.jsonl",
"exists": true
},
{
"path": "scripts/run_output_eval.py",
"exists": true
},
{
"path": "scripts/run_output_execution.py",
"exists": true
},
{
"path": "reports/output_quality_scorecard.json",
"exists": true
},
{
"path": "tests/verify_output_eval_lab.py",
"exists": true
}
],
"next_action": "Add more real-file and adversarial holdout cases as adoption data grows."
},
{
"key": "runtime-conformance",
"category": "core-module",
"label": "Runtime Conformance",
"status": "pass",
"objective": "Target packages can be consumed by OpenAI, Claude, Agent Skills, VS Code, and generic targets.",
"current": "5/5 targets pass",
"command": "python3 scripts/yao.py conformance .",
"test": "python3 tests/verify_conformance_suite.py",
"evidence": [
{
"path": "runtime/conformance/schema.json",
"exists": true
},
{
"path": "scripts/run_conformance_suite.py",
"exists": true
},
{
"path": "reports/conformance_matrix.json",
"exists": true
},
{
"path": "tests/verify_conformance_suite.py",
"exists": true
}
],
"next_action": "Keep conformance cases aligned with current platform metadata rules."
},
{
"key": "trust-security",
"category": "core-module",
"label": "Trust Security",
"status": "pass",
"objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.",
"current": "155 scripts; secrets 0; help failures 0",
"command": "python3 scripts/yao.py trust .",
"test": "python3 tests/verify_trust_check.py",
"evidence": [
{
"path": "scripts/trust_check.py",
"exists": true
},
{
"path": "security/trust_policy.md",
"exists": true
},
{
"path": "security/script_policy.md",
"exists": true
},
{
"path": "security/permission_policy.json",
"exists": true
},
{
"path": "reports/security_trust_report.json",
"exists": true
},
{
"path": "tests/verify_trust_check.py",
"exists": true
}
],
"next_action": "Keep high-permission approvals scoped, expiring, and mapped to target enforcement."
},
{
"key": "skill-atlas",
"category": "core-module",
"label": "Skill Atlas",
"status": "pass",
"objective": "Team skill portfolio reveals route collisions, stale ownership, dependency graph, and no-route opportunities.",
"current": "11 scanned skills; actionable collisions 0",
"command": "python3 scripts/yao.py skill-atlas --workspace-root .",
"test": "python3 tests/verify_skill_atlas.py",
"evidence": [
{
"path": "scripts/build_skill_atlas.py",
"exists": true
},
{
"path": "skill_atlas/catalog.json",
"exists": true
},
{
"path": "skill_atlas/route_overlap_matrix.csv",
"exists": true
},
{
"path": "skill_atlas/dependency_graph.json",
"exists": true
},
{
"path": "reports/skill_atlas.json",
"exists": true
},
{
"path": "tests/verify_skill_atlas.py",
"exists": true
}
],
"next_action": "Use real telemetry to rank stale or drifting skills by impact, not only by static metadata."
},
{
"key": "registry-distribution",
"category": "core-module",
"label": "Registry Distribution",
"status": "pass",
"objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.",
"current": "archive entries 580; install failures 0",
"command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .",
"test": "python3 tests/verify_registry_audit.py",
"evidence": [
{
"path": "registry/index.schema.json",
"exists": true
},
{
"path": "registry/package.schema.json",
"exists": true
},
{
"path": "registry/packages/yao-meta-skill.json",
"exists": true
},
{
"path": "scripts/registry_audit.py",
"exists": true
},
{
"path": "reports/package_verification.json",
"exists": true
},
{
"path": "reports/install_simulation.json",
"exists": true
},
{
"path": "tests/verify_registry_audit.py",
"exists": true
}
],
"next_action": "Regenerate registry metadata after package verification so source and archive checksums stay aligned."
},
{
"key": "review-studio",
"category": "core-module",
"label": "Review Studio",
"status": "pass",
"objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.",
"current": "16 gates; decision review; warnings 3",
"command": "python3 scripts/yao.py review-studio .",
"test": "python3 tests/verify_review_studio.py",
"evidence": [
{
"path": "scripts/render_review_studio.py",
"exists": true
},
{
"path": "reports/review-studio.html",
"exists": true
},
{
"path": "reports/review-studio.json",
"exists": true
},
{
"path": "tests/verify_review_studio.py",
"exists": true
}
],
"next_action": "Close pending human and external evidence before claiming full release readiness."
},
{
"key": "telemetry-drift",
"category": "core-module",
"label": "Telemetry Drift",
"status": "pass",
"objective": "Real usage feedback is captured as metadata-only local-first drift signals.",
"current": "events 1; recipes 5; risk low",
"command": "python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .",
"test": "python3 tests/verify_telemetry_hooks.py",
"evidence": [
{
"path": "scripts/emit_telemetry_event.py",
"exists": true
},
{
"path": "scripts/import_telemetry_events.py",
"exists": true
},
{
"path": "scripts/telemetry_native_host.py",
"exists": true
},
{
"path": "reports/adoption_drift_report.json",
"exists": true
},
{
"path": "reports/telemetry_hook_recipes.json",
"exists": true
},
{
"path": "tests/verify_telemetry_hooks.py",
"exists": true
}
],
"next_action": "Install a real client and import production metadata-only events into the local drift loop."
}
],
"recommended_prs": [
{
"key": "benchmark-methodology",
"category": "recommended-pr",
"label": "Benchmark Methodology",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "25 required artifacts checked",
"command": "make ci-test",
"test": "tests/verify_benchmark_reproducibility.py",
"evidence": [
{
"path": "reports/benchmark_methodology.md",
"exists": true
},
{
"path": "reports/benchmark_reproducibility.json",
"exists": true
},
{
"path": "tests/verify_benchmark_reproducibility.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "output-eval-schema",
"category": "recommended-pr",
"label": "Output Eval Schema",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "5 output cases",
"command": "make ci-test",
"test": "tests/verify_output_eval_lab.py",
"evidence": [
{
"path": "evals/output/schema.json",
"exists": true
},
{
"path": "evals/output/cases.jsonl",
"exists": true
},
{
"path": "tests/verify_output_eval_lab.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "output-eval-runner",
"category": "recommended-pr",
"label": "Output Eval Runner",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "delta 100.0",
"command": "make ci-test",
"test": "tests/verify_output_eval_lab.py",
"evidence": [
{
"path": "scripts/run_output_eval.py",
"exists": true
},
{
"path": "reports/output_quality_scorecard.json",
"exists": true
},
{
"path": "tests/verify_output_eval_lab.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "output-quality-scorecard",
"category": "recommended-pr",
"label": "Output Quality Scorecard",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "gate pass True",
"command": "make ci-test",
"test": "tests/verify_output_eval_lab.py",
"evidence": [
{
"path": "reports/output_quality_scorecard.md",
"exists": true
},
{
"path": "reports/output_quality_scorecard.json",
"exists": true
},
{
"path": "tests/verify_output_eval_lab.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "skill-ir-v0",
"category": "recommended-pr",
"label": "Skill IR V0",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "schema 2.0.0",
"command": "make ci-test",
"test": "tests/verify_skill_ir.py",
"evidence": [
{
"path": "skill-ir/schema.json",
"exists": true
},
{
"path": "skill-ir/examples/yao-meta-skill.json",
"exists": true
},
{
"path": "tests/verify_skill_ir.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "compiler-refactor",
"category": "recommended-pr",
"label": "Compiler Refactor",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "5/5 compiled targets",
"command": "make ci-test",
"test": "tests/verify_compile_skill.py",
"evidence": [
{
"path": "scripts/compile_skill.py",
"exists": true
},
{
"path": "reports/compiled_targets.json",
"exists": true
},
{
"path": "tests/verify_compile_skill.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "agent-skills-conformance",
"category": "recommended-pr",
"label": "Agent Skills Conformance",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "agent-skills target present",
"command": "make ci-test",
"test": "tests/verify_conformance_suite.py",
"evidence": [
{
"path": "runtime/conformance/schema.json",
"exists": true
},
{
"path": "reports/conformance_matrix.json",
"exists": true
},
{
"path": "tests/verify_conformance_suite.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "trust-check",
"category": "recommended-pr",
"label": "Trust Check",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "secret findings 0",
"command": "make ci-test",
"test": "tests/verify_trust_check.py",
"evidence": [
{
"path": "scripts/trust_check.py",
"exists": true
},
{
"path": "reports/security_trust_report.json",
"exists": true
},
{
"path": "tests/verify_trust_check.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "skill-atlas-generator",
"category": "recommended-pr",
"label": "Skill Atlas Generator",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "11 scanned skills",
"command": "make ci-test",
"test": "tests/verify_skill_atlas.py",
"evidence": [
{
"path": "scripts/build_skill_atlas.py",
"exists": true
},
{
"path": "reports/skill_atlas.json",
"exists": true
},
{
"path": "tests/verify_skill_atlas.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "registry-package-format",
"category": "recommended-pr",
"label": "Registry Package Format",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "registry ok True",
"command": "make ci-test",
"test": "tests/verify_registry_audit.py",
"evidence": [
{
"path": "registry/package.schema.json",
"exists": true
},
{
"path": "reports/registry_audit.json",
"exists": true
},
{
"path": "tests/verify_registry_audit.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "review-studio-2",
"category": "recommended-pr",
"label": "Review Studio 2.0",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "16 review gates",
"command": "make ci-test",
"test": "tests/verify_review_studio.py",
"evidence": [
{
"path": "scripts/render_review_studio.py",
"exists": true
},
{
"path": "reports/review-studio.html",
"exists": true
},
{
"path": "tests/verify_review_studio.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "migration-v2-docs",
"category": "recommended-pr",
"label": "Migration V2 Docs",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "migration guide present",
"command": "make ci-test",
"test": "docs review",
"evidence": [
{
"path": "docs/migration-v2.md",
"exists": true
},
{
"path": "reports/skill-os-2-review.md",
"exists": true
},
{
"path": "README.md",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
},
{
"key": "evidence-consistency",
"category": "recommended-pr",
"label": "Evidence Consistency",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "39 consistency checks",
"command": "make ci-test",
"test": "tests/verify_evidence_consistency.py",
"evidence": [
{
"path": "scripts/render_evidence_consistency.py",
"exists": true
},
{
"path": "reports/evidence_consistency.json",
"exists": true
},
{
"path": "tests/verify_evidence_consistency.py",
"exists": true
}
],
"next_action": "Keep this item covered as the implementation evolves."
}
],
"reference_extension_tracks": [
{
"key": "skill-interpretation-report",
"label": "Skill Interpretation Report",
"status": "covered",
"objective": "User-facing deep interpretation report explains use cases, triggers, inputs, outputs, workflow, principles, boundaries, quality gates, examples, and next iterations.",
"current": "Skill Overview v2 is canonical and mirrored as first-class skill-interpretation HTML/JSON with schema and tests.",
"target": "Either keep skill-overview as the canonical interpretation report with an explicit contract, or split a dedicated reports/skill-interpretation.* renderer and tests.",
"evidence": [
{
"path": "reports/skill-overview.html",
"exists": true
},
{
"path": "reports/skill-overview.json",
"exists": true
},
{
"path": "scripts/render_skill_overview.py",
"exists": true
},
{
"path": "scripts/render_skill_interpretation.py",
"exists": true
},
{
"path": "schemas/skill-interpretation.schema.json",
"exists": true
},
{
"path": "tests/verify_skill_interpretation.py",
"exists": true
}
],
"next_action": "Keep overview and interpretation contracts in lockstep when report sections, metrics, or layout semantics change."
},
{
"key": "adaptive-self-iteration",
"label": "Adaptive Self-Iteration",
"status": "covered",
"objective": "Local-first preference memory, repeated-signal extraction, adaptation proposals, approval, patch application, regression evidence, and rollback.",
"current": "Full adaptive loop includes proposal, approval, patch application, regression evidence, and rollback metadata.",
"target": "Proposal-only adaptation with explicit input source, redaction, allowlisted write targets, approval ledger, regression report, and rollback plan.",
"evidence": [
{
"path": "references/autonomous-adaptation.md",
"exists": true
},
{
"path": "references/user-memory-policy.md",
"exists": true
},
{
"path": "schemas/adaptation-proposal.schema.json",
"exists": true
},
{
"path": "scripts/summarize_user_signals.py",
"exists": true
},
{
"path": "scripts/propose_adaptation.py",
"exists": true
},
{
"path": "tests/verify_adaptation_safety.py",
"exists": true
},
{
"path": "scripts/apply_adaptation.py",
"exists": true
},
{
"path": "reports/adaptation_approval_ledger.json",
"exists": true
},
{
"path": "reports/adaptation_regression_report.json",
"exists": true
},
{
"path": "reports/user_patterns.json",
"exists": true
},
{
"path": "reports/adaptation_proposals.json",
"exists": true
},
{
"path": "reports/iteration-directions.md",
"exists": true
},
{
"path": "reports/adoption_drift_report.md",
"exists": true
}
],
"next_action": "Start with policy and read-only scan tests; do not read shell history or private logs unless the user provides an explicit source path."
},
{
"key": "daily-skillops-report",
"label": "Daily SkillOps Report",
"status": "covered",
"objective": "Daily operations layer summarizes explicit-source conversation patterns, proposal-only adaptation work, approval state, release locks, and world-class evidence gaps.",
"current": "Daily SkillOps report is a CLI-backed, explicit-source operations cockpit with tests and committed dated evidence.",
"target": "Generate reports/skillops/daily/YYYY-MM-DD.* from explicit sources without private log scanning, source writes, auto-patching, or world-class overclaiming.",
"evidence": [
{
"path": "scripts/render_daily_skillops_report.py",
"exists": true
},
{
"path": "tests/verify_daily_skillops.py",
"exists": true
},
{
"path": "reports/skillops/daily/2026-07-02.json",
"exists": true
},
{
"path": "reports/skillops/daily/2026-07-02.md",
"exists": true
}
],
"next_action": "Keep Daily SkillOps report aligned with proposal, approval, coverage, and world-class ledger contracts as the operations layer evolves."
},
{
"key": "weekly-curator-report",
"label": "Weekly Curator Report",
"status": "covered",
"objective": "Weekly curator layer aggregates Daily SkillOps opportunities, Skill Atlas portfolio signals, release locks, and world-class evidence gaps into a maintenance queue.",
"current": "Weekly curator report is CLI-backed, proposal-only, and backed by committed weekly JSON/Markdown evidence.",
"target": "Generate reports/skillops/weekly/YYYY-WNN.* without private log scanning, source writes, auto-patching, or world-class overclaiming.",
"evidence": [
{
"path": "scripts/render_weekly_curator_report.py",
"exists": true
},
{
"path": "tests/verify_weekly_curator.py",
"exists": true
},
{
"path": "reports/skillops/weekly/2026-W27.json",
"exists": true
},
{
"path": "reports/skillops/weekly/2026-W27.md",
"exists": true
}
],
"next_action": "Use weekly curator output as the Skill Librarian maintenance queue before approving any durable skill-library changes."
}
],
"next_highest_leverage": [
"Close the four world-class evidence ledger entries with accepted human or external evidence.",
"Keep the first-class skill interpretation report and Skill Overview v2 contract synchronized as the report model evolves.",
"Use Daily SkillOps with explicit curated sources to review adaptation proposals without scanning private logs or auto-patching.",
"Keep the blueprint coverage report in CI so 2.0 plan drift is visible."
],
"source_blueprint": {
"title": "Skill Overview / Skill OS 2.0 upgrade plan",
"core_module_count": 8,
"recommended_pr_count": 13,
"reference_extension_count": 4,
"reference_extensions": [
"Skill interpretation report",
"Adaptive self-iteration with local preference memory and approval gates",
"Daily SkillOps operations report"
]
},
"artifacts": {
"json": "reports/skill_os2_coverage.json",
"markdown": "reports/skill_os2_coverage.md"
}
}